Skip to content

new config setting - margo.server_address #821

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

new config setting - margo.server_address #821

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions common/src/unifyfs_configurator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
UNIFYFS_CFG(margo, client_pool_size, INT, UNIFYFS_MARGO_POOL_SZ, "size of server's ULT pool for client-server RPCs", NULL) \
UNIFYFS_CFG(margo, client_timeout, INT, UNIFYFS_MARGO_CLIENT_SERVER_TIMEOUT_MSEC, "timeout in milliseconds for client-server RPCs", NULL) \
UNIFYFS_CFG(margo, lazy_connect, BOOL, on, "wait until first communication with server to resolve its connection address", NULL) \
UNIFYFS_CFG(margo, server_address, STRING, NULLSTRING, "Mercury address string to use for Margo init of server-server communication", NULL) \
UNIFYFS_CFG(margo, server_pool_size, INT, UNIFYFS_MARGO_POOL_SZ, "size of server's ULT pool for server-server RPCs", NULL) \
UNIFYFS_CFG(margo, server_timeout, INT, UNIFYFS_MARGO_SERVER_SERVER_TIMEOUT_MSEC, "timeout in milliseconds for server-server RPCs", NULL) \
UNIFYFS_CFG(margo, tcp, BOOL, on, "use TCP for server-to-server margo RPCs", NULL) \
Expand Down
15 changes: 8 additions & 7 deletions docs/configuration.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,14 @@ files.
.. table:: ``[margo]`` section - margo server NA settings
:widths: auto

============== ==== =================================================================================
Key Type Description
============== ==== =================================================================================
tcp BOOL Use TCP for server-to-server rpcs (default: on, turn off to enable libfabric RMA)
client_timeout INT timeout in milliseconds for rpcs between client and server (default: 5000)
server_timeout INT timeout in milliseconds for rpcs between servers (default: 15000)
============== ==== =================================================================================
============== ====== ====================================================================================
Key Type Description
============== ====== ====================================================================================
tcp BOOL Use TCP for server-to-server rpcs (default: on, turn off to enable libfabric RMA)
client_timeout INT timeout in milliseconds for rpcs between client and server (default: 5000)
server_timeout INT timeout in milliseconds for rpcs between servers (default: 15000)
server_address STRING Mercury address string to use for Margo initialization of inter-server communication
============== ====== ====================================================================================


-----------
Expand Down
32 changes: 20 additions & 12 deletions server/src/margo_server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@

// global variables
ServerRpcContext_t* unifyfsd_rpc_context;
char* margo_init_address; // = NULL
bool margo_use_address; // = false
bool margo_use_tcp = true;
bool margo_lazy_connect; // = false
int margo_client_server_pool_sz = UNIFYFS_MARGO_POOL_SZ;
Expand Down Expand Up @@ -99,31 +101,37 @@ static char* get_margo_addr_str(margo_instance_id mid)
/* setup_remote_target - Initializes the server-server margo target */
static margo_instance_id setup_remote_target(void)
{
/* by default we try to use ofi */
const char* margo_protocol = margo_use_tcp ?
const char* server_addr = NULL;

if (margo_use_address) {
server_addr = margo_init_address;
} else {
/* by default we try to use ofi */
server_addr = margo_use_tcp ?
PROTOCOL_MARGO_OFI_TCP : PROTOCOL_MARGO_OFI_RMA;
if (!margo_protocol) {
/* when ofi is not available, fallback to using bmi */
LOGWARN("OFI is not available, using BMI for margo rpc");
margo_protocol = PROTOCOL_MARGO_BMI_TCP;
if (!server_addr) {
/* when ofi is not available, fallback to using bmi */
LOGWARN("OFI is not available, using BMI for margo rpc");
server_addr = PROTOCOL_MARGO_BMI_TCP;
}
}

/* initialize margo */
margo_instance_id mid = margo_init(margo_protocol, MARGO_SERVER_MODE,
margo_instance_id mid = margo_init(server_addr, MARGO_SERVER_MODE,
margo_use_progress_thread, margo_server_server_pool_sz);
if (mid == MARGO_INSTANCE_NULL) {
LOGERR("margo_init(%s, SERVER_MODE, %d, %d) failed",
margo_protocol, margo_use_progress_thread,
server_addr, margo_use_progress_thread,
margo_server_server_pool_sz);
if (margo_protocol == PROTOCOL_MARGO_OFI_TCP) {
if (server_addr == PROTOCOL_MARGO_OFI_TCP) {
/* try "ofi+sockets" instead */
margo_protocol = PROTOCOL_MARGO_OFI_SOCKETS;
mid = margo_init(margo_protocol, MARGO_SERVER_MODE,
server_addr = PROTOCOL_MARGO_OFI_SOCKETS;
mid = margo_init(server_addr, MARGO_SERVER_MODE,
margo_use_progress_thread,
margo_server_server_pool_sz);
if (mid == MARGO_INSTANCE_NULL) {
LOGERR("margo_init(%s, SERVER_MODE, %d, %d) failed",
margo_protocol, margo_use_progress_thread,
server_addr, margo_use_progress_thread,
margo_server_server_pool_sz);
return mid;
}
Expand Down
3 changes: 2 additions & 1 deletion server/src/margo_server.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ typedef struct ServerRpcContext {
} ServerRpcContext_t;

extern ServerRpcContext_t* unifyfsd_rpc_context;

extern char* margo_init_address;
extern bool margo_use_address;
extern bool margo_use_tcp;
extern bool margo_lazy_connect;
extern int margo_client_server_pool_sz;
Expand Down
13 changes: 10 additions & 3 deletions server/src/unifyfs_server.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,9 +427,16 @@ int main(int argc, char* argv[])
margo_lazy_connect = b;
}

rc = configurator_bool_val(server_cfg.margo_tcp, &b);
if (0 == rc) {
margo_use_tcp = b;
if (server_cfg.margo_server_address != NULL) {
margo_use_address = true;
margo_init_address = strdup(server_cfg.margo_server_address);
}

if (!margo_use_address) {
rc = configurator_bool_val(server_cfg.margo_tcp, &b);
if (0 == rc) {
margo_use_tcp = b;
}
}

rc = margo_server_rpc_init();
Expand Down