Skip to content

Commit 47539fa

Browse files
ernst-bablickernst-bablick
authored
Several changes related to Python API (#40)
* EB: CS-857: Create pybind11 compatible GCS library * EH: CS-875: Rewrite version module, make information available vi C++ class and expose it to python EH: CS-869: Embed python interpreter in qmaster and enable it in all * EH: CS-889: Add a new class that represents a DataStore on client side and make it available for python. * EH: CS-890: PyCharm code completion not working for the ocs-bridge * EH: CS-890: PyCharm code completion not working for the ocs-bridge * EH: CS-290: add a build step that allows to generate CULL header from JSON files * EH: CS-290: add a build step that allows to generate CULL header from JSON files * EH: CS-290: add a build step that allows to generate CULL header from JSON files * Fixed issue reported by github workflow * Fixed issue reported by github workflow
1 parent c592390 commit 47539fa

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1334
-1071
lines changed

‎CMakeLists.txt

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ option(WITH_MTMALLOC "Enable use of the mtmalloc memory allocator on Solaris" ON
6464
option(WITH_QMAKE "Enable build of qmake" ON)
6565
option(WITH_JNI "Add JNI code for libraries like libdrmaa" ON)
6666
option(WITH_GPERF "Enable profiling code with Google Performance Tools" OFF)
67+
option(WITH_PYTHON "Enable Python external bindings" OFF)
6768

6869
# private extensions
6970
set(PROJECT_EXTENSIONS "None" CACHE STRING "directory of private extensions")
@@ -159,6 +160,8 @@ if (WITH_GPERF)
159160
set(GPERFTOOLS_PROFILER "") # set to empty string to avoid linking against gperftools
160161
message("Cannot find gperftools although WITH_GPERF is set")
161162
endif()
163+
else()
164+
set(GPERFTOOLS_PROFILER "")
162165
endif()
163166

164167
if (WITH_HWLOC)
@@ -196,6 +199,48 @@ if (WITH_JNI)
196199
endif()
197200
endif()
198201

202+
if (WITH_PYTHON)
203+
# Ensure that we have Python3 installed
204+
find_package(Python3 COMPONENTS Interpreter Development)
205+
if (Python3_FOUND)
206+
message(STATUS "Python3 found: ${Python3_EXECUTABLE}")
207+
message(STATUS "Python3 include dirs: ${Python3_INCLUDE_DIRS}")
208+
message(STATUS "Python3 libraries: ${Python3_LIBRARIES}")
209+
message(STATUS "Python3 version: ${Python3_VERSION}")
210+
else()
211+
message(FATAL_ERROR "Python3 not found")
212+
endif()
213+
214+
# We need pybind11 (usually installed via 'pip install pybind11')
215+
# Find the cmake directory of pybind so that find_package will work
216+
execute_process(COMMAND python3 -m pybind11 --cmakedir OUTPUT_VARIABLE PYBIND11_CMAKE_DIR)
217+
string(STRIP ${PYBIND11_CMAKE_DIR} pybind11_DIR)
218+
find_package(pybind11 REQUIRED)
219+
if (NOT pybind11_FOUND)
220+
message(FATAL_ERROR "pybind11 not found")
221+
else()
222+
# python3 -m pybind11 --includes
223+
message(STATUS "pybind11 include dirs: ${pybind11_INCLUDE_DIR}")
224+
endif ()
225+
226+
# find the python/pybind11 extension suffix (e.g .cpython-310-x86_64-linux-gnu.so)
227+
execute_process(COMMAND python3 -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))" OUTPUT_VARIABLE PYBIND11_EXT_SUFFIX)
228+
string(STRIP ${PYBIND11_EXT_SUFFIX} PYBIND11_EXTENSION_SUFFIX)
229+
message(STATUS "Python3 ext-suffix: ${PYBIND11_EXTENSION_SUFFIX}")
230+
231+
# find the pybind11-stubgen tool that allows us to generate pyi-files
232+
# pyi-files are used by IDEs to provide type hints for the pybind11 module
233+
#find_program(PYBIND11_STUBGEN pybind11-stubgen)
234+
find_program(PYBIND11_STUBGEN stubgen)
235+
if (PYBIND11_STUBGEN)
236+
message(STATUS "Python stubgen to generate pyi: ${PYBIND11_STUBGEN}")
237+
endif()
238+
239+
set(PYTHON_ALL_INCLUDE_DIRS ${Python3_INCLUDE_DIRS} ${pybind11_INCLUDE_DIR})
240+
set(PYTHON_ALL_LIBRARIES ${Python3_LIBRARIES} util)
241+
set(PYTHON_EXTENSION_SUFFIX ${PYBIND11_EXTENSION_SUFFIX})
242+
endif ()
243+
199244
if (SGE_ARCH MATCHES "fbsd-amd64")
200245
set(SGE_LIBS_SHARED pthread ${TIRPC_LIB} kvm ${CMAKE_DL_LIBS} m)
201246
else ()

‎cmake/ArchitectureSpecificSettings.cmake

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@ function(architecture_specific_settings)
6464
if (SGE_ARCH MATCHES "lx-riscv64")
6565
# Linux RiscV
6666
message(STATUS "We are on Linux: ${SGE_ARCH}")
67-
# -Wno-deprecated-declarations
6867
set(CMAKE_C_FLAGS "-Wall -Werror -pedantic" CACHE STRING "" FORCE)
6968
set(CMAKE_CXX_FLAGS "-Wall -Werror -pedantic" CACHE STRING "" FORCE)
7069

@@ -80,6 +79,13 @@ function(architecture_specific_settings)
8079
set(WITH_MTMALLOC OFF PARENT_SCOPE)
8180
set(JNI_ARCH "linux" PARENT_SCOPE)
8281
elseif (SGE_ARCH MATCHES "lx-.*" OR SGE_ARCH MATCHES "ulx-.*" OR SGE_ARCH MATCHES "xlx-.*")
82+
# Disable Python for unsupported qmaster architectures
83+
if (WITH_PYTHON)
84+
if (SGE_ARCH MATCHES "lx-ppc64le" OR SGE_ARCH MATCHES "lx-s390x")
85+
set(WITH_PYTHON OFF PARENT_SCOPE)
86+
endif()
87+
endif()
88+
8389
# Linux supported/unsupported amd64/x86
8490
message(STATUS "We are on Linux: ${SGE_ARCH}")
8591
set(CMAKE_C_FLAGS "-Wall -Werror -pedantic" CACHE STRING "" FORCE)
@@ -108,12 +114,13 @@ function(architecture_specific_settings)
108114
# specific linux architectures
109115
if (SGE_TARGETBITS STREQUAL "TARGET_32BIT")
110116
add_compile_definitions(_FILE_OFFSET_BITS=64)
111-
# readdir64_r seems to be deprecated
117+
# readdir64_r seems to be deprecated CS-199
112118
add_compile_options(-Wno-deprecated-declarations)
113119
elseif ((OS_ID STREQUAL "raspbian" AND OS_VERSION EQUAL 10)
114120
OR (OS_ID STREQUAL "tuxedo" AND OS_VERSION EQUAL 22.04)
115121
OR (OS_ID STREQUAL "ubuntu" AND OS_VERSION EQUAL 22.04)
116122
OR (OS_ID STREQUAL "rocky" AND OS_VERSION EQUAL 9.4))
123+
# readdir64_r seems to be deprecated CS-199
117124
add_compile_options(-Wno-deprecated-declarations)
118125
endif ()
119126

@@ -143,6 +150,8 @@ function(architecture_specific_settings)
143150
set(TIRPC_INCLUDES /usr/include/ntirpc PARENT_SCOPE)
144151
set(TIRPC_LIB ntirpc PARENT_SCOPE)
145152
message(STATUS "using libntirpc")
153+
else ()
154+
message(STATUS "no libtirpc or libntirpc found")
146155
endif ()
147156

148157
if (SGE_ARCH STREQUAL "lx-x86" OR SGE_ARCH STREQUAL "ulx-x86" OR SGE_ARCH STREQUAL "xlx-x86")
@@ -169,6 +178,9 @@ function(architecture_specific_settings)
169178

170179
set(JNI_ARCH "linux" PARENT_SCOPE)
171180
elseif (SGE_ARCH MATCHES "fbsd-amd64")
181+
# Disable Python for unsupported qmaster architectures
182+
set(WITH_PYTHON OFF PARENT_SCOPE)
183+
172184
# FreeBSD
173185
message(STATUS "We are on FreeBSD: ${SGE_ARCH}")
174186
set(PROJECT_AUTOMAKE_SRC "/usr/local/share/automake-*/config.*" PARENT_SCOPE)
@@ -183,6 +195,9 @@ function(architecture_specific_settings)
183195

184196
set(JNI_ARCH "freebsd" PARENT_SCOPE)
185197
elseif (SGE_ARCH MATCHES "sol-.*")
198+
# Disable Python for unsupported qmaster architectures
199+
set(WITH_PYTHON OFF PARENT_SCOPE)
200+
186201
# Solaris
187202
message(STATUS "We are on Solaris: ${SGE_ARCH}")
188203
add_compile_definitions(SOLARIS GETHOSTBYNAME_R5 GETHOSTBYADDR_R7 SPOOLING_dynamic __SGE_COMPILE_WITH_GETTEXT__)
@@ -192,6 +207,9 @@ function(architecture_specific_settings)
192207

193208
set(JNI_ARCH "solaris" PARENT_SCOPE)
194209
elseif (SGE_ARCH MATCHES "darwin-arm64")
210+
# Disable Python for unsupported qmaster architectures
211+
set(WITH_PYTHON OFF PARENT_SCOPE)
212+
195213
# Darwin M1/M2/M2Max/M2Pro (arm64) platform
196214
message(STATUS "We are on macOS: ${SGE_ARCH}")
197215
# -Wextra

‎doc/markdown/manual/release-notes/02_os_matrix.md

Lines changed: 30 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,38 +5,41 @@ xxQS_NAMExx components on the other axis.
55

66
| Operating System | Version | Architecture | Master/Shadow Service | Execution Service | Admin/Submit Clients |
77
|:-----------------|:--------|:-------------|:---------------------:|:-----------------:|:--------------------:|
8-
| macOS | 14 | ARM64 | a | a | x |
9-
| Rocky Linux | 8 | ARM64 | x | x | x |
10-
| Rocky Linux | 9 | ARM64 | x | x | x |
11-
| Raspbian Linux | 11 | ARM64 | x | x | x |
12-
| Raspbian Linux | 12 | ARM64 | x | x | x |
8+
| macOS | 14 | ARM64 | - | a | a |
9+
| Rocky Linux | 8 | ARM64 | x | x | x |
10+
| Rocky Linux | 9 | ARM64 | x | x | x |
11+
| Raspbian Linux | 11 | ARM64 | x | x | x |
12+
| Raspbian Linux | 12 | ARM64 | x | x | x |
1313
| | | | | | |
14-
| Alma Linux | 8 | ppc64le | x | x | x |
15-
| Centos Linux | 8 | ppc64le | x | x | x |
16-
| Rocky Linux | 8 | ppc64le | x | x | x |
14+
| Alma Linux | 8 | ppc64le | a | x | x |
15+
| Centos Linux | 8 | ppc64le | a | x | x |
16+
| Rocky Linux | 8 | ppc64le | a | x | x |
1717
| | | | | | |
18-
| SUSE Tumbleweed | | Risc-V64 | a | x | x |
18+
| SUSE Tumbleweed | | Risc-V64 | a | x | x |
1919
| | | | | | |
20-
| Alma Linux | 8 | x86-64 | x | x | x |
21-
| Alma Linux | 9 | x86-64 | x | x | x |
22-
| CentOS Linux | 7 | x86-64 | x | x | x |
23-
| CentOS Linux | 8 | x86-64 | x | x | x |
24-
| CentOS Linux | 9 | x86-64 | x | x | x |
25-
| Free BSD | 13 | x86-64 | a | x | x |
26-
| Free BSD | 14 | x86-64 | a | x | x |
27-
| Redhat Linux | 8 | x86-64 | x | x | x |
28-
| Redhat Linux | 9 | x86-64 | x | x | x |
29-
| Rocky Linux | 8 | x86-64 | x | x | x |
30-
| Rocky Linux | 9 | x86-64 | x | x | x |
31-
| Solaris | 11 | x86-64 | x | x | x |
32-
| SUSE Leap Linux | 15 | x86-64 | x | x | x |
33-
| SUSE Tumbleweed | | x86-64 | a | x | x |
34-
| Ubuntu Linux | 20.04 | x86-64 | x | x | x |
35-
| Ubuntu Linux | 22.04 | x86-64 | x | x | x |
36-
| Ubuntu Linux | 24.04 | x86-64 | x | x | x |
20+
| Alma Linux | 8 | x86-64 | x | x | x |
21+
| Alma Linux | 9 | x86-64 | x | x | x |
22+
| CentOS Linux | 6 | x86-64 | a | a | a |
23+
| CentOS Linux | 7 | x86-64 | x | x | x |
24+
| CentOS Linux | 8 | x86-64 | x | x | x |
25+
| CentOS Linux | 9 | x86-64 | x | x | x |
26+
| Free BSD | 13 | x86-64 | a | x | x |
27+
| Free BSD | 14 | x86-64 | a | x | x |
28+
| Redhat Linux | 6 | x86-64 | a | a | a |
29+
| Redhat Linux | 7 | x86-64 | a | x | x |
30+
| Redhat Linux | 8 | x86-64 | x | x | x |
31+
| Redhat Linux | 9 | x86-64 | x | x | x |
32+
| Rocky Linux | 8 | x86-64 | x | x | x |
33+
| Rocky Linux | 9 | x86-64 | x | x | x |
34+
| Solaris | 11 | x86-64 | - | a | x |
35+
| SUSE Leap Linux | 15 | x86-64 | x | x | x |
36+
| SUSE Tumbleweed | | x86-64 | a | x | x |
37+
| Ubuntu Linux | 20.04 | x86-64 | x | x | x |
38+
| Ubuntu Linux | 22.04 | x86-64 | x | x | x |
39+
| Ubuntu Linux | 24.04 | x86-64 | x | x | x |
3740

3841
-: Unsupported
39-
a: Available but still not supported
42+
a: Available but not supported. Contact our sales and support team if you need this configuration
4043
x: Supported
4144

4245
> **Note**

‎source/CMakeLists.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,15 @@
1919
#___INFO__MARK_END_NEW__
2020

2121
# source
22+
23+
# sgeobj library needs to be first, as it contains the generation of CULL header files required by others
24+
add_subdirectory(libs)
25+
2226
add_subdirectory(3rdparty)
2327
add_subdirectory(clients)
2428
add_subdirectory(common)
2529
add_subdirectory(daemons)
2630
add_subdirectory(dist)
27-
add_subdirectory(libs)
2831
add_subdirectory(security)
2932
add_subdirectory(utilbin)
3033

‎source/clients/common/ocs_client_job.cc

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
#include "uti/sge_rmon_macros.h"
4444
#include "uti/sge_time.h"
4545

46-
#include "sgeobj/cull/sge_all_listsL.h"
4746
#include "sgeobj/cull_parse_util.h"
4847
#include "sgeobj/ocs_binding_io.h"
4948
#include "sgeobj/sge_centry.h"
@@ -54,6 +53,13 @@
5453
#include "sgeobj/sge_ulong.h"
5554
#include "sgeobj/sge_usage.h"
5655
#include "sgeobj/sge_var.h"
56+
#include "sgeobj/sge_str.h"
57+
#include "sgeobj/sge_path_alias.h"
58+
#include "sgeobj/sge_qref.h"
59+
#include "sgeobj/sge_ja_task.h"
60+
#include "sgeobj/sge_pe_task.h"
61+
#include "sgeobj/sge_mesobj.h"
62+
#include "sgeobj/sge_grantedres.h"
5763

5864
#include "get_path.h"
5965
#include "ocs_client_job.h"

‎source/clients/common/ocs_client_print.cc

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,6 @@
4343
#include "uti/sge_time.h"
4444
#include "uti/sge_unistd.h"
4545

46-
#include "sgeobj/cull/sge_all_listsL.h"
4746
#include "sgeobj/parse.h"
4847
#include "sgeobj/sge_host.h"
4948
#include "sgeobj/sge_job.h"

‎source/common/parse_qsub.cc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@
4545
#include "uti/sge_string.h"
4646
#include "uti/sge_time.h"
4747

48-
#include "sgeobj/cull/sge_all_listsL.h"
4948
#include "sgeobj/cull_parse_util.h"
5049
#include "sgeobj/sge_mailrec.h"
5150
#include "sgeobj/parse.h"
@@ -60,6 +59,7 @@
6059
#include "sgeobj/sge_answer.h"
6160
#include "sgeobj/sge_jsv.h"
6261
#include "sgeobj/sge_qref.h"
62+
#include "sgeobj/sge_binding.h"
6363

6464
#include "parse_job_cull.h"
6565
#include "parse_qsub.h"

‎source/daemons/execd/load_avg.cc

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
#include "sgeobj/sge_object.h"
6666
#include "sgeobj/sge_usage.h"
6767

68-
#include "gdi/version.h"
68+
#include "sgeobj/ocs_Version.h"
6969
#include "sgeobj/sge_daemonize.h"
7070
#include "gdi/sge_gdiP.h"
7171

@@ -221,7 +221,7 @@ execd_add_load_report(lList *report_list, u_long64 now, u_long64 *next_send)
221221
} else {
222222
lSetUlong(report, REP_type, NUM_REP_REPORT_LOAD);
223223
}
224-
lSetUlong(report, REP_version, GRM_GDI_VERSION);
224+
lSetUlong(report, REP_version, ocs::Version::get_version());
225225
lSetUlong(report, REP_seqno, sge_execd_report_seqno);
226226
lSetHost(report, REP_host, qualified_hostname);
227227

@@ -306,7 +306,7 @@ execd_add_conf_report(lList *report_list, u_long64 now, u_long64 *next_send)
306306
*/
307307
report = lCreateElem(REP_Type);
308308
lSetUlong(report, REP_type, NUM_REP_REPORT_CONF);
309-
lSetUlong(report, REP_version, GRM_GDI_VERSION);
309+
lSetUlong(report, REP_version, ocs::Version::get_version());
310310
lSetUlong(report, REP_seqno, sge_execd_report_seqno);
311311
lSetHost(report, REP_host, qualified_hostname);
312312
lSetList(report, REP_list,
@@ -332,7 +332,7 @@ execd_add_license_report(lList *report_list, u_long64 now, u_long64 *next_send)
332332
*/
333333
report = lCreateElem(REP_Type);
334334
lSetUlong(report, REP_type, NUM_REP_REPORT_PROCESSORS);
335-
lSetUlong(report, REP_version, GRM_GDI_VERSION);
335+
lSetUlong(report, REP_version, ocs::Version::get_version());
336336
lSetUlong(report, REP_seqno, sge_execd_report_seqno);
337337
lSetHost(report, REP_host, qualified_hostname);
338338
{
@@ -396,7 +396,7 @@ execd_add_job_report(lList *report_list, u_long64 now, u_long64 *next_send)
396396
/* create job report */
397397
job_report = lCreateElem(REP_Type);
398398
lSetUlong(job_report, REP_type, NUM_REP_REPORT_JOB);
399-
lSetUlong(job_report, REP_version, GRM_GDI_VERSION);
399+
lSetUlong(job_report, REP_version, ocs::Version::get_version());
400400
lSetUlong(job_report, REP_seqno, sge_execd_report_seqno);
401401
lSetHost(job_report, REP_host, qualified_hostname);
402402

0 commit comments

Comments
 (0)