Skip to content

Commit 9183f85

Browse files
mohitkhullarmohitkhullar
committed
enforce max call time
Signed-off-by: mohitkhullar <[email protected]>
1 parent df37a80 commit 9183f85

File tree

2 files changed

+92
-28
lines changed

2 files changed

+92
-28
lines changed

‎cdb2api/cdb2api.c‎

Lines changed: 91 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,9 @@ static int cdb2_comdb2db_timeout_set_from_env = 0;
131131
static int CDB2_API_CALL_TIMEOUT = 120000; /* defaults to 2 minute */
132132
static int cdb2_api_call_timeout_set_from_env = 0;
133133

134+
static int CDB2_ENFORCE_API_CALL_TIMEOUT = 0;
135+
static int cdb2_enforce_api_call_timeout_set_from_env = 0;
136+
134137
static int CDB2_SOCKET_TIMEOUT = 5000;
135138
static int cdb2_socket_timeout_set_from_env = 0;
136139

@@ -856,6 +859,34 @@ static int is_sql_read(const char *sqlstr)
856859
#define HAVE_MSGHDR_MSG_CONTROL
857860
#endif
858861

862+
static int is_api_call_timedout(cdb2_hndl_tp *hndl) {
863+
struct timeval tv;
864+
gettimeofday(&tv, NULL);
865+
long long current_time = tv.tv_sec*1000 + tv.tv_usec/1000;
866+
if (hndl->max_call_time && (hndl->max_call_time < current_time)) {
867+
return 1;
868+
}
869+
return 0;
870+
}
871+
872+
static long long get_call_timeout(const cdb2_hndl_tp *hndl, long long timeout) {
873+
struct timeval tv;
874+
gettimeofday(&tv, NULL);
875+
long long current_time = tv.tv_sec*1000 + tv.tv_usec/1000;
876+
long long time_left = hndl->max_call_time - current_time;
877+
if (hndl->max_call_time && time_left <= 0)
878+
time_left = 1;
879+
if (time_left > 0 && (time_left < timeout))
880+
return time_left;
881+
return timeout;
882+
}
883+
884+
static void set_max_call_time(cdb2_hndl_tp *hndl) {
885+
struct timeval tv;
886+
gettimeofday(&tv, NULL);
887+
hndl->max_call_time = tv.tv_sec*1000 + tv.tv_usec/1000 + hndl->api_call_timeout;
888+
}
889+
859890
enum {
860891
PASSFD_SUCCESS = 0,
861892
PASSFD_RECVMSG = -1, /* error with recvmsg() */
@@ -995,8 +1026,7 @@ static int recv_fd_int(int sockfd, void *data, size_t nbytes, int *fd_recvd, int
9951026
static int recv_fd(const cdb2_hndl_tp *hndl, int sockfd, void *data, size_t nbytes, int *fd_recvd)
9961027
{
9971028
int rc, timeoutms = hndl ? hndl->sockpool_recv_timeoutms : CDB2_SOCKPOOL_RECV_TIMEOUTMS;
998-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
999-
timeoutms = hndl->api_call_timeout;
1029+
timeoutms = get_call_timeout(hndl, timeoutms);
10001030
rc = recv_fd_int(sockfd, data, nbytes, fd_recvd, timeoutms);
10011031
if (rc != 0 && *fd_recvd != -1) {
10021032
int errno_save = errno;
@@ -1116,8 +1146,7 @@ static int send_fd_to(int sockfd, const void *data, size_t nbytes,
11161146
static int send_fd(const cdb2_hndl_tp *hndl, int sockfd, const void *data, size_t nbytes, int fd_to_send)
11171147
{
11181148
int timeoutms = hndl ? hndl->sockpool_send_timeoutms : CDB2_SOCKPOOL_SEND_TIMEOUTMS;
1119-
if (hndl && hndl->api_call_timeout && (timeoutms > hndl->api_call_timeout))
1120-
timeoutms = hndl->api_call_timeout;
1149+
timeoutms = get_call_timeout(hndl, timeoutms);
11211150
return send_fd_to(sockfd, data, nbytes, fd_to_send, timeoutms);
11221151
}
11231152

@@ -1591,6 +1620,8 @@ static void read_comdb2db_environment_cfg(cdb2_hndl_tp *hndl, const char *comdb2
15911620
&cdb2_sockpool_recv_timeoutms_set_from_env);
15921621
process_env_var_int("COMDB2_CONFIG_API_CALL_TIMEOUT", &CDB2_API_CALL_TIMEOUT,
15931622
&cdb2_api_call_timeout_set_from_env);
1623+
process_env_var_int("COMDB2_CONFIG_ENFORCE_API_CALL_TIMEOUT", &CDB2_ENFORCE_API_CALL_TIMEOUT,
1624+
&cdb2_enforce_api_call_timeout_set_from_env);
15941625
process_env_var_int("COMDB2_CONFIG_COMDB2DB_TIMEOUT", &COMDB2DB_TIMEOUT, &cdb2_comdb2db_timeout_set_from_env);
15951626
process_env_var_int("COMDB2_CONFIG_SOCKET_TIMEOUT", &CDB2_SOCKET_TIMEOUT, &cdb2_socket_timeout_set_from_env);
15961627
process_env_var_int("COMDB2_CONFIG_PROTOBUF_SIZE", &CDB2_PROTOBUF_SIZE, &cdb2_protobuf_size_set_from_env);
@@ -1841,6 +1872,9 @@ static void read_comdb2db_cfg(cdb2_hndl_tp *hndl, SBUF2 *s, const char *comdb2db
18411872
hndl->api_call_timeout = atoi(tok);
18421873
else if (tok)
18431874
CDB2_API_CALL_TIMEOUT = atoi(tok);
1875+
} else if (!cdb2_api_call_timeout_set_from_env && (strcasecmp("enforce_api_call_timeout",tok) == 0)) {
1876+
tok = strtok_r(NULL, " :,", &last);
1877+
CDB2_ENFORCE_API_CALL_TIMEOUT = value_on_off(tok, &err);
18441878
} else if (strcasecmp("auto_consume_timeout", tok) == 0) {
18451879
tok = strtok_r(NULL, " :,", &last);
18461880
if (tok)
@@ -2125,6 +2159,8 @@ static void set_cdb2_timeouts(cdb2_hndl_tp *hndl)
21252159
hndl->comdb2db_timeout = hndl->api_call_timeout;
21262160
if (hndl->socket_timeout > hndl->api_call_timeout)
21272161
hndl->socket_timeout = hndl->api_call_timeout;
2162+
2163+
set_max_call_time(hndl);
21282164
}
21292165

21302166
/* Read all available comdb2 configuration files.
@@ -3688,8 +3724,22 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
36883724

36893725
debugprint("name %s\n", name);
36903726

3727+
int connect_timeout = hndl->connect_timeout;
3728+
3729+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3730+
# ifdef CDB2API_TEST
3731+
printf("RETRY with timeout %d\n", get_call_timeout(hndl, connect_timeout));
3732+
# endif
3733+
if (is_api_call_timedout(hndl)) {
3734+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3735+
port = -1;
3736+
goto after_callback;
3737+
}
3738+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3739+
}
3740+
36913741
fd = cdb2_tcpconnecth_to(hndl, remote_host, CDB2_PORTMUXPORT, 0,
3692-
hndl->connect_timeout);
3742+
connect_timeout);
36933743
if (fd < 0) {
36943744
debugprint("cdb2_tcpconnecth_to returns fd=%d'\n", fd);
36953745
if (errno == EINPROGRESS) {
@@ -3715,7 +3765,18 @@ static int cdb2portmux_get(cdb2_hndl_tp *hndl, const char *type,
37153765
port = -1;
37163766
goto after_callback;
37173767
}
3718-
sbuf2settimeout(ss, hndl->connect_timeout, hndl->connect_timeout);
3768+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3769+
# ifdef CDB2API_TEST
3770+
printf("RETRY with timeout %d\n", get_call_timeout(hndl, connect_timeout));
3771+
# endif
3772+
if (is_api_call_timedout(hndl)) {
3773+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out connecting to db\n", __func__, __LINE__);
3774+
port = -1;
3775+
goto after_callback;
3776+
}
3777+
connect_timeout = get_call_timeout(hndl, connect_timeout);
3778+
}
3779+
sbuf2settimeout(ss, connect_timeout, connect_timeout);
37193780
sbuf2printf(ss, "get %s\n", name);
37203781
sbuf2flush(ss);
37213782
res[0] = 0;
@@ -3904,6 +3965,17 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
39043965
goto after_callback;
39053966

39063967
retry:
3968+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
3969+
int socket_timeout = get_call_timeout(hndl, hndl->socket_timeout);
3970+
# ifdef CDB2API_TEST
3971+
printf("GOT HEARTBEAT || Set timeout to %d\n", socket_timeout);
3972+
# endif
3973+
if (is_api_call_timedout(hndl)) {
3974+
snprintf(hndl->errstr, sizeof(hndl->errstr), "%s:%d Timed out reading response from the db\n", __func__, __LINE__);
3975+
rc = -1;
3976+
}
3977+
sbuf2settimeout(sb, socket_timeout, socket_timeout);
3978+
}
39073979
b_read = sbuf2fread((char *)&hdr, 1, sizeof(hdr), sb);
39083980
debugprint("READ HDR b_read=%d, sizeof(hdr)=(%zu):\n", b_read, sizeof(hdr));
39093981

@@ -4068,6 +4140,10 @@ static int cdb2_read_record(cdb2_hndl_tp *hndl, uint8_t **buf, int *len, int *ty
40684140

40694141
rc = 0;
40704142
after_callback:
4143+
// reset here
4144+
if (CDB2_ENFORCE_API_CALL_TIMEOUT) {
4145+
sbuf2settimeout(sb, hndl->socket_timeout, hndl->socket_timeout);
4146+
}
40714147
while ((e = cdb2_next_callback(hndl, CDB2_AFTER_READ_RECORD, e)) != NULL) {
40724148
callbackrc =
40734149
cdb2_invoke_callback(hndl, e, 1, CDB2_RETURN_VALUE, (intptr_t)rc);
@@ -5954,10 +6030,8 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
59546030
hndl->retry_all = 1;
59556031
int run_last = 1;
59566032

5957-
time_t max_time =
5958-
time(NULL) + (hndl->api_call_timeout - hndl->connect_timeout) / 1000;
5959-
if (max_time < 0)
5960-
max_time = 0;
6033+
set_max_call_time(hndl);
6034+
59616035
retry_queries:
59626036
debugprint(
59636037
"retry_queries: hndl->host=%d (%s)\n", hndl->connected_host,
@@ -5985,18 +6059,11 @@ static int cdb2_run_statement_typed_int(cdb2_hndl_tp *hndl, const char *sql, int
59856059
cdb2_get_dbhosts(hndl);
59866060
}
59876061

5988-
int tmsec = 0;
5989-
5990-
// Add wait if we have already tried on all the nodes.
5991-
if (!hndl->sb && (retries_done > hndl->num_hosts)) {
5992-
tmsec = (retries_done - hndl->num_hosts) * 100;
5993-
}
5994-
59956062
if (hndl->sslerr != 0)
59966063
PRINT_AND_RETURN(CDB2ERR_CONNECT_ERROR);
59976064

59986065
if ((retries_done > 1) && ((retries_done > hndl->max_retries) ||
5999-
((time(NULL) + (tmsec / 1000)) >= max_time))) {
6066+
(is_api_call_timedout(hndl)))) {
60006067
sprintf(hndl->errstr, "%s: Maximum number of retries done.", __func__);
60016068
if (is_hasql_commit) {
60026069
cleanup_query_list(hndl, &commit_query_list, __LINE__);
@@ -7873,17 +7940,13 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
78737940
}
78747941
}
78757942

7876-
time_t max_time =
7877-
time(NULL) +
7878-
(hndl->api_call_timeout - (CDB2_POLL_TIMEOUT + hndl->connect_timeout)) /
7879-
1000;
7880-
if (max_time < 0)
7881-
max_time = 0;
7943+
if (!hndl->max_call_time)
7944+
set_max_call_time(hndl);
78827945

78837946
use_bmsd = cdb2_use_bmsd && (*cdb2_bmssuffix != '\0') && !hndl->num_shards; // cannot find shards via bmsd yet
78847947
retry:
78857948
if (rc) {
7886-
if (num_retry >= MAX_RETRIES || time(NULL) > max_time)
7949+
if (num_retry >= MAX_RETRIES || is_api_call_timedout(hndl))
78877950
goto after_callback;
78887951

78897952
num_retry++;
@@ -7901,7 +7964,7 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79017964
hndl, cdb2_default_cluster, comdb2db_name, comdb2db_num,
79027965
comdb2db_hosts[i], comdb2db_hosts, comdb2db_ports, &master,
79037966
&num_comdb2db_hosts, NULL);
7904-
if (rc == 0 || time(NULL) >= max_time) {
7967+
if (rc == 0 || is_api_call_timedout(hndl)) {
79057968
break;
79067969
}
79077970
}
@@ -7918,15 +7981,15 @@ static int cdb2_get_dbhosts(cdb2_hndl_tp *hndl)
79187981
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->cluster, &hndl->dbnum,
79197982
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,
79207983
&hndl->num_shards_sameroom);
7921-
if (rc == 0 || time(NULL) >= max_time) {
7984+
if (rc == 0 || is_api_call_timedout(hndl)) {
79227985
break;
79237986
} else if (use_bmsd) {
79247987
if (cdb2_comdb2db_fallback)
79257988
use_bmsd = 0;
79267989
goto retry;
79277990
}
79287991
}
7929-
if (rc == -1 && time(NULL) < max_time) {
7992+
if (rc == -1 && !is_api_call_timedout(hndl)) {
79307993
rc = comdb2db_get_dbhosts(hndl, comdb2db_name, comdb2db_num, comdb2db_hosts[master], comdb2db_ports[master],
79317994
hndl->hosts, &hndl->num_hosts, hndl->dbname, hndl->cluster, &hndl->dbnum,
79327995
&hndl->num_hosts_sameroom, num_retry, use_bmsd, hndl->shards, &hndl->num_shards,

‎cdb2api/cdb2api_hndl.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,7 @@ struct cdb2_hndl {
169169
int is_hasql;
170170
int sent_client_info;
171171
void *user_arg;
172+
long long max_call_time;
172173
int api_call_timeout;
173174
int connect_timeout;
174175
int comdb2db_timeout;

0 commit comments

Comments
 (0)