Skip to content

Commit 37a4494

Browse files
mohitkhullarmohitkhullar
committed
enforce max call time
Signed-off-by: mohitkhullar <[email protected]>
1 parent df37a80 commit 37a4494

File tree

2 files changed

+95
-28
lines changed

2 files changed

+95
-28
lines changed

‎cdb2api/cdb2api.c‎

Lines changed: 94 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ static int cdb2_comdb2db_timeout_set_from_env = 0;
131131
static int CDB2_API_CALL_TIMEOUT = 120000; /* defaults to 2 minute */
132132
static int cdb2_api_call_timeout_set_from_env = 0;
133133

134+
static int CDB2_ENFORCE_API_CALL_TIMEOUT = 0;
135+
static int cdb2_enforce_api_call_timeout_set_from_env = 0;
136+
134137
static int CDB2_SOCKET_TIMEOUT = 5000;
135138
static int cdb2_socket_timeout_set_from_env = 0;
136139

@@ -856,6 +859,37 @@ static int is_sql_read(const char *sqlstr)
856859
#define HAVE_MSGHDR_MSG_CONTROL
857860
#endif
858861

862+
static int is_api_call_timedout(cdb2_hndl_tp *hndl) {
863+
struct timeval tv;
864+
gettimeofday(&tv, NULL);
865+
long long current_time = tv.tv_sec*1000 + tv.tv_usec/1000;
866+
if (hndl->max_call_time && (hndl->max_call_time < current_time)) {
867+
return 1;
868+
}
869+
return 0;
870+
}
871+
872+
static long long get_call_timeout(const cdb2_hndl_tp *hndl, long long timeout) {
873+
if (!hndl)
874+
return timeout;
875+
struct timeval tv;
876+
gettimeofday(&tv, NULL);
877+
long long current_time = tv.tv_sec*1000 + tv.tv_usec/1000;
878+
long long time_left = hndl->max_call_time - current_time;
879+
if (hndl->max_call_time && time_left <= 0)
880+
time_left = 1;
881+
if (time_left > 0 && (time_left < timeout))
882+
return time_left;
883+
return timeout;
884+
}
885+
886+
static void set_max_call_time(cdb2_hndl_tp *hndl) {
887+
struct timeval tv;
888+
gettimeofday(&tv, NULL);
889+
if (hndl)
890+
hndl->max_call_time = tv.tv_sec*1000 + tv.tv_usec/1000 + hndl->api_call_timeout;
891+
}
892+
859893
enum {
860894
PASSFD_SUCCESS = 0,
861895
PASSFD_RECVMSG = -1, /* error with recvmsg() */
@@ -995,8 +1029,7 @@ static int recv_fd_int(int sockfd, void *data, size_t nbytes, int *fd_recvd, int
9951029
static int recv_fd(const cdb2_hndl_tp *hndl, int sockfd, void *data, size_t nbytes, int *fd_recvd)
9961030
{
9971031
int rc, timeoutms = hndl ? hndl->sockpool_recv_timeoutms : CDB2_SOCKPOOL_RECV_TIMEOUTMS;
998-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
999-
timeoutms = hndl->api_call_timeout;
1032+
timeoutms = get_call_timeout(hndl, timeoutms);
10001033
rc = recv_fd_int(sockfd, data, nbytes, fd_recvd, timeoutms);
10011034
if (rc != 0 && *fd_recvd != -1) {
10021035
int errno_save = errno;
@@ -1116,8 +1149,7 @@ static int send_fd_to(int sockfd, const void *data, size_t nbytes,
11161149
static int send_fd(const cdb2_hndl_tp *hndl, int sockfd, const void *data, size_t nbytes, int fd_to_send)
11171150
{
11181151
int timeoutms = hndl ? hndl->sockpool_send_timeoutms : CDB2_SOCKPOOL_SEND_TIMEOUTMS;
1119-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
1120-
timeoutms = hndl->api_call_timeout;
1152+
timeoutms = get_call_timeout(hndl, timeoutms);
11211153
return send_fd_to(sockfd, data, nbytes, fd_to_send, timeoutms);
11221154
}
11231155

@@ -1591,6 +1623,8 @@ static void read_comdb2db_environment_cfg(cdb2_hndl_tp *hndl, const char *comdb2
15911623
&cdb2_sockpool_recv_timeoutms_set_from_env);
15921624
process_env_var_int("COMDB2_CONFIG_API_CALL_TIMEOUT", &CDB2_API_CALL_TIMEOUT,
15931625
&cdb2_api_call_timeout_set_from_env);
1626+
process_env_var_int("COMDB2_CONFIG_ENFORCE_API_CALL_TIMEOUT", &CDB2_ENFORCE_API_CALL_TIMEOUT,
1627+
&cdb2_enforce_api_call_timeout_set_from_env);
15941628
process_env_var_int("COMDB2_CONFIG_COMDB2DB_TIMEOUT", &COMDB2DB_TIMEOUT, &cdb2_comdb2db_timeout_set_from_env);
15951629
process_env_var_int("COMDB2_CONFIG_SOCKET_TIMEOUT", &CDB2_SOCKET_TIMEOUT, &cdb2_socket_timeout_set_from_env);
15961630
process_env_var_int("COMDB2_CONFIG_PROTOBUF_SIZE", &CDB2_PROTOBUF_SIZE, &cdb2_protobuf_size_set_from_env);
@@ -1841,6 +1875,9 @@ static void read_comdb2db_cfg(cdb2_hndl_tp *hndl, SBUF2 *s, const char *comdb2db
18411875
hndl->api_call_timeout = atoi(tok);
18421876
else if (tok)
18431877
CDB2_API_CALL_TIMEOUT = atoi(tok);
1878+
} else if (!cdb2_api_call_timeout_set_from_env && (strcasecmp("enforce_api_call_timeout",tok) == 0)) {
1879+
tok = strtok_r(NULL, " :,", &last);
1880+
CDB2_ENFORCE_API_CALL_TIMEOUT = value_on_off(tok, &err);
18441881
} else if (strcasecmp("auto_consume_timeout", tok) == 0) {
18451882
tok = strtok_r(NULL, " :,", &last);
18461883
if (tok)
@@ -2125,6 +2162,8 @@ static void set_cdb2_timeouts(cdb2_hndl_tp *hndl)
21252162
hndl->comdb2db_timeout = hndl->api_call_timeout;
21262163
if (hndl->socket_timeout > hndl->api_call_timeout)
21272164
hndl->socket_timeout = hndl->api_call_timeout;
2165+
2166+
set_max_call_time(hndl);
21282167
}
21292168

21302169
/* Read all available comdb2 configuration files.
@@ -3688,8 +3727,22 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
36883727

36893728
debugprint("name %s\n", name);
36903729

3730+
int connect_timeout = hndl->connect_timeout;
3731+
3732+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3733+
# ifdef CDB2API_TEST
3734+
printf("RETRY with timeout %d\n", get_call_timeout(hndl, connect_timeout));
3735+
# endif
3736+
if (is_api_call_timedout(hndl)) {
3737+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3738+
port = -1;
3739+
goto after_callback;
3740+
}
3741+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3742+
}
3743+
36913744
fd = cdb2_tcpconnecth_to(hndl, remote_host, CDB2_PORTMUXPORT, 0,
3692-
hndl->connect_timeout);
3745+
connect_timeout);
36933746
if (fd < 0) {
36943747
debugprint("cdb2_tcpconnecth_to returns fd=%d'\n", fd);
36953748
if (errno == EINPROGRESS) {
@@ -3715,7 +3768,18 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
37153768
port = -1;
37163769
goto after_callback;
37173770
}
3718-
sbuf2settimeout(ss, hndl->connect_timeout, hndl->connect_timeout);
3771+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3772+
# ifdef CDB2API_TEST
3773+
printf("RETRY with timeout %d\n", get_call_timeout(hndl, connect_timeout));
3774+
# endif
3775+
if (is_api_call_timedout(hndl)) {
3776+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3777+
port = -1;
3778+
goto after_callback;
3779+
}
3780+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3781+
}
3782+
sbuf2settimeout(ss, connect_timeout, connect_timeout);
37193783
sbuf2printf(ss, "get %s\n", name);
37203784
sbuf2flush(ss);
37213785
res[0] = 0;
@@ -3904,6 +3968,17 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
39043968
goto after_callback;
39053969

39063970
retry:
3971+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3972+
int socket_timeout = get_call_timeout(hndl, hndl->socket_timeout);
3973+
# ifdef CDB2API_TEST
3974+
printf("GOT HEARTBEAT || Set timeout to %d\n", socket_timeout);
3975+
# endif
3976+
if (is_api_call_timedout(hndl)) {
3977+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out reading response from the db\n", __func__, __LINE__);
3978+
rc = -1;
3979+
}
3980+
sbuf2settimeout(sb, socket_timeout, socket_timeout);
3981+
}
39073982
b_read = sbuf2fread((char *)&hdr, 1, sizeof(hdr), sb);
39083983
debugprint("READ HDR b_read=%d, sizeof(hdr)=(%zu):\n", b_read, sizeof(hdr));
39093984

@@ -4068,6 +4143,10 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
40684143

40694144
rc = 0;
40704145
after_callback:
4146+
// reset here
4147+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
4148+
sbuf2settimeout(sb, hndl->socket_timeout, hndl->socket_timeout);
4149+
}
40714150
while ((e = cdb2_next_callback(hndl, CDB2_AFTER_READ_RECORD, e)) != NULL) {
40724151
callbackrc =
40734152
cdb2_invoke_callback(hndl, e, 1, CDB2_RETURN_VALUE, (intptr_t)rc);
@@ -5954,10 +6033,8 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
59546033
hndl->retry_all = 1;
59556034
int run_last = 1;
59566035

5957-
time_t max_time =
5958-
time(NULL) + (hndl->api_call_timeout - hndl->connect_timeout) / 1000;
5959-
if (max_time < 0)
5960-
max_time = 0;
6036+
set_max_call_time(hndl);
6037+
59616038
retry_queries:
59626039
debugprint(
59636040
"retry_queries: hndl->host=%d (%s)\n", hndl->connected_host,
@@ -5985,18 +6062,11 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
59856062
cdb2_get_dbhosts(hndl);
59866063
}
59876064

5988-
int tmsec = 0;
5989-
5990-
// Add wait if we have already tried on all the nodes.
5991-
if (!hndl->sb && (retries_done > hndl->num_hosts)) {
5992-
tmsec = (retries_done - hndl->num_hosts) * 100;
5993-
}
5994-
59956065
if (hndl->sslerr != 0)
59966066
PRINT_AND_RETURN(CDB2ERR_CONNECT_ERROR);
59976067

59986068
if ((retries_done > 1) && ((retries_done > hndl->max_retries) ||
5999-
((time(NULL) + (tmsec / 1000)) >= max_time))) {
6069+
(is_api_call_timedout(hndl)))) {
60006070
sprintf(hndl->errstr, "%s: Maximum number of retries done.", __func__);
60016071
if (is_hasql_commit) {
60026072
cleanup_query_list(hndl, &commit_query_list, __LINE__);
@@ -7873,17 +7943,13 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
78737943
}
78747944
}
78757945

7876-
time_t max_time =
7877-
time(NULL) +
7878-
(hndl->api_call_timeout - (CDB2_POLL_TIMEOUT + hndl->connect_timeout)) /
7879-
1000;
7880-
if (max_time < 0)
7881-
max_time = 0;
7946+
if (!hndl->max_call_time)
7947+
set_max_call_time(hndl);
78827948

78837949
use_bmsd = cdb2_use_bmsd && (*cdb2_bmssuffix != '\0') && !hndl->num_shards; // cannot find shards via bmsd yet
78847950
retry:
78857951
if (rc) {
7886-
if (num_retry >= MAX_RETRIES || time(NULL) > max_time)
7952+
if (num_retry >= MAX_RETRIES || is_api_call_timedout(hndl))
78877953
goto after_callback;
78887954

78897955
num_retry++;
@@ -7901,7 +7967,7 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79017967
hndl, cdb2_default_cluster, comdb2db_name, comdb2db_num,
79027968
comdb2db_hosts[i], comdb2db_hosts, comdb2db_ports, &master,
79037969
&num_comdb2db_hosts, NULL);
7904-
if (rc == 0 || time(NULL) >= max_time) {
7970+
if (rc == 0 || is_api_call_timedout(hndl)) {
79057971
break;
79067972
}
79077973
}
@@ -7918,15 +7984,15 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79187984
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->cluster, &hndl->dbnum,
79197985
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,
79207986
&hndl->num_shards_sameroom);
7921-
if (rc == 0 || time(NULL) >= max_time) {
7987+
if (rc == 0 || is_api_call_timedout(hndl)) {
79227988
break;
79237989
} else if (use_bmsd) {
79247990
if (cdb2_comdb2db_fallback)
79257991
use_bmsd = 0;
79267992
goto retry;
79277993
}
79287994
}
7929-
if (rc == -1 && time(NULL) < max_time) {
7995+
if (rc == -1 && !is_api_call_timedout(hndl)) {
79307996
rc = comdb2db_get_dbhosts(hndl, comdb2db_name, comdb2db_num, comdb2db_hosts[master], comdb2db_ports[master],
79317997
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->cluster, &hndl->dbnum,
79327998
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,

‎cdb2api/cdb2api_hndl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ struct cdb2_hndl {
169169
int is_hasql;
170170
int sent_client_info;
171171
void *user_arg;
172+
long long max_call_time;
172173
int api_call_timeout;
173174
int connect_timeout;
174175
int comdb2db_timeout;

0 commit comments

Comments
 (0)