iser: support rdma ack timeout optimization

Since 2c1619edef61a03cb516efaa81750784c3071d10 for linux kernel and
55843c4ab8f559679d28c559cc4d681836be769b for rdma-core, rdma cma
supports RDMA_OPTION_ID_ACK_TIMEOUT. It's useful for RDMA out of
sequence case. Because this feature is added recently, we have to
check this in autogen.sh before building source code.

Depend on production enviroument, tunning rdma ack timeout could get
the best performance. Suggested by Bart and Ronnie, instead of using
a fixed timeout value, add two methods to set rdma ack timeout value.
1, add URL variable 'LIBISCSI_RDMA_ACK_TIMEOUT'. This could works
for a specified connection.
2, add env argument 'LIBISCSI_RDMA_ACK_TIMEOUT'. This works as a
common setting for all the connection of a process.

Test under different packet loss rate and different ack timeout, run
fio (iodepth=1) in a guest os, I got this result:
latency under packet loss rate 0.00001:
	timeout 19: avg 170.22, pct99.9 215
	timeout 10: avg 160.08, pct99.9 215
	timeout 8 : avg 146.39, pct99.9 177
	timeout 7 : avg 148.37, pct99.9 211

latency under packet loss rate 0.0001:
	timeout 19: avg 949.23, pct99.9 306
	timeout 10: avg 818.53, pct99.9 378
	timeout 8 : avg 615.84, pct99.9 189
	timeout 7 : avg 618.89, pct99.9 310

Base on this test report, setting ack timeout to 8(1048.576 usec) is
a good choice in my test enviroument.

Signed-off-by: zhenwei pi <pizhenwei@bytedance.com>

configure.ac

@@ -152,6 +152,11 @@ if test x"$libiscsi_cv_HAVE_LINUX_ISER" = x"yes"; then
 fi
 AM_CONDITIONAL([HAVE_LINUX_ISER], [test $libiscsi_cv_HAVE_LINUX_ISER = yes])
 
+AC_TRY_COMPILE([
+#include <rdma/rdma_cma.h>],
+[return RDMA_OPTION_ID_ACK_TIMEOUT;],
+[AC_DEFINE([HAVE_RDMA_ACK_TIMEOUT],[1],[Define to 1 if you have RDMA ack timeout support])],)
+
 AC_CACHE_CHECK([whether libcunit is available],
                [ac_cv_have_cunit],
                [ac_save_CFLAGS="$CFLAGS"

include/iscsi-private.h

@@ -109,6 +109,8 @@ struct iscsi_context {
 	int tcp_syncnt;
 	int tcp_nonblocking;
 
+	unsigned char rdma_ack_timeout;
+
 	int current_phase;
 	int next_phase;
 #define ISCSI_LOGIN_SECNEG_PHASE_OFFER_CHAP         0

lib/init.c

@@ -280,6 +280,10 @@ iscsi_create_context(const char *initiator_name)
 		iscsi_set_bind_interfaces(iscsi,getenv("LIBISCSI_BIND_INTERFACES"));
 	}
 
+	if (getenv("LIBISCSI_RDMA_ACK_TIMEOUT") != NULL) {
+		iscsi->rdma_ack_timeout = (unsigned char)atoi(getenv("LIBISCSI_RDMA_ACK_TIMEOUT"));
+	}
+
 	/* iscsi->smalloc_size is the size for small allocations. this should be
 	   max(ISCSI_HEADER_SIZE, sizeof(struct iscsi_pdu), sizeof(struct iscsi_in_pdu))
 	   rounded up to the next power of 2. */
@@ -617,6 +621,8 @@ iscsi_parse_url(struct iscsi_context *iscsi, const char *url, int full)
 #ifdef HAVE_LINUX_ISER
 			} else if (!strcmp(key, "iser")) {
 				is_iser = 1;
+			} else if (!strcmp(key, "LIBISCSI_RDMA_ACK_TIMEOUT")) {
+				iscsi->rdma_ack_timeout = (unsigned char)atoi(value);
 #endif
 			}
 			tmp = next;

lib/iser.c

@@ -15,6 +15,10 @@
    along with this program; if not, see <http://www.gnu.org/licenses/>.
 */
 
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/prctl.h>
@@ -812,6 +816,40 @@ static int iser_create_iser_conn_res(struct iser_conn *iser_conn) {
 	return ret;
 }
 
+void iscsi_set_rdma_ack_timetout(struct iscsi_context *iscsi, unsigned char value)
+{
+	iscsi->rdma_ack_timeout = value;
+	ISCSI_LOG(iscsi, 2, "RDMA ACK TIMEOUT will be set to %d on next rdma qp creation", value);
+}
+
+/*
+ * iser_rdma_set_option() - set rdma options
+ * currently support RDMA ack timeout.
+ * @cma_id:    connection manager id
+ * @iscsi:   iscsi context
+ */
+int iser_rdma_set_option(struct rdma_cm_id *cma_id, struct iscsi_context *iscsi) {
+	int ret = 0;
+	unsigned char timeout = iscsi->rdma_ack_timeout;
+
+	if (timeout) {
+#ifdef HAVE_RDMA_ACK_TIMEOUT
+		/* calculate according to the formula 4.096 * 2^(timeout) usec.
+		   Ex, 8 means 1048.576 usec (0.00104 sec) */
+		ret = rdma_set_option(cma_id, RDMA_OPTION_ID, RDMA_OPTION_ID_ACK_TIMEOUT, &timeout, sizeof(timeout));
+		if (ret) {
+			ISCSI_LOG(iscsi, 1, "failed to set RDMA ACK TIMEOUT: %s", strerror(errno));
+		} else {
+			ISCSI_LOG(iscsi, 3, "RDMA ACK TIMEOUT set to %d", timeout);
+		}
+#else
+		ISCSI_LOG(iscsi, 1, "failed to set RDMA ACK TIMEOUT(%d) for cma_id(%p), please update your kernel&librdmacm", timeout, cma_id);
+#endif
+	}
+
+	return ret;
+}
+
 /*
  * iser_addr_handler() - handles RDMA_CM_EVENT_ADDR_RESOLVED
  *                       event in rdma_cm
@@ -823,6 +861,8 @@ static int iser_addr_handler(struct rdma_cm_id *cma_id) {
 	struct iser_conn *iser_conn = iscsi->opaque;
 	int ret, flags;
 
+	iser_rdma_set_option(cma_id, iscsi);
+
 	ret = rdma_resolve_route(cma_id, 1000);
 	if (ret) {
 		iscsi_set_error(iscsi, "Failed resolving address\n");