Hi,

We would like to add SDP support for OPENMPI.

SDP - Socket Direct Protocol is a byte-stream transport protocol implementing the TCP SOCK_STREAM semantics utilizing transport offloading capabilities of the InfiniBand fabric

(http://www.mellanox.com/pdf/whitepapers/SDP_Whitepaper.pdf, http://www.openfabrics.org/archives/aug2005datacenter/das_SDP_Linux.pdf ).

SDP can be used to accelerate job start ( oob over sdp ) and IPoIB performance.

The main idea is to use AF_INET_SDP protocol family instead of AF_INET and AF_INET6 when opening sockets.

SDP will be used in OOB and BTL with appropriate mca parameters –

-mca btl_tcp_sdp_enable 1

-mca oob_tcp_sdp_enable 1

Since not all functions support this family, the changes were maid only in critical sections of the code

Since SDP support is relevant only for InfiniBand Fabrics you need to configure sdp support with –enable-sdp flag. SDP will be disabled by default.

./configure –enable-sdp

Test results of running bandwidth and latency of SDP on 2 DDR nodes.

BWsize VERBS IPoIB connected IPoIB datagram btl SDP

1000000 1507.68 665.70 425.21 1272.37

LTsize

5 3.82 28.83 28.24 25.73

Index: opal/include/opal_config_bottom.h

===================================================================

--- opal/include/opal_config_bottom.h (revision 17027)

+++ opal/include/opal_config_bottom.h (working copy)

@@ -509,7 +509,15 @@

#if !HAVE_DECL_PF_INET6

#define PF_INET6 PF_UNSPEC

#endif

+#if !HAVE_DECL_AF_INET_SDP

+#define AF_INET_SDP 27

+#endif

+#if OPAL_ENABLE_SDP

+#define OPAL_WANT_SDP 1

+#else

+#define OPAL_WANT_SDP 0

+#endif

#if defined(__APPLE__) && defined(HAVE_INTTYPES_H)

/* Prior to Mac OS X 10.3, the length modifier "ll" wasn't

supported, but "q" was for long long. This isn't ANSI

Index: configure.ac

===================================================================

--- configure.ac (revision 17027)

+++ configure.ac (working copy)

@@ -674,7 +674,7 @@

#include <netinet/in.h>

#endif])

-AC_CHECK_DECLS([AF_UNSPEC, PF_UNSPEC, AF_INET6, PF_INET6],

+AC_CHECK_DECLS([AF_UNSPEC, PF_UNSPEC, AF_INET6, PF_INET6, AF_INET_SDP],

[], [], [AC_INCLUDES_DEFAULT

#if HAVE_SYS_SOCKET_H

#include <sys/socket.h>

Index: ompi/mca/btl/tcp/btl_tcp_component.c

===================================================================

--- ompi/mca/btl/tcp/btl_tcp_component.c (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp_component.c (working copy)

@@ -263,6 +263,10 @@

mca_btl_tcp_component.tcp_disable_family =

mca_btl_tcp_param_register_int ("disable_family", NULL, 0);

+#if OPAL_WANT_SDP

+ mca_btl_tcp_component.sdp_enable =

+ mca_btl_tcp_param_register_int("sdp_enable", "Enable SDP for TCP connections", 0);

+#endif

return OMPI_SUCCESS;

}

@@ -527,6 +531,11 @@

memset (&hints, 0, sizeof(hints));

hints.ai_family = af_family;

+#if OPAL_WANT_SDP

+ if ( mca_btl_tcp_component.sdp_enable ) {

+ hints.ai_family = AF_INET6;

+ }

+#endif

hints.ai_socktype = SOCK_STREAM;

hints.ai_flags = AI_PASSIVE;

@@ -555,7 +564,7 @@

#endif /* IPV6_V6ONLY */

}

#else

- ((struct sockaddr_in*) &inaddr)->sin_family = AF_INET;

+ ((struct sockaddr_in*) &inaddr)->sin_family = af_family;

((struct sockaddr_in*) &inaddr)->sin_addr.s_addr = INADDR_ANY;

addrlen = sizeof(struct sockaddr_in);

#endif

@@ -600,7 +609,11 @@

}

goto socket_binded;

}

- if( AF_INET == af_family ) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

BTL_ERROR(("bind() failed: no port available in the range [%d..%d]",

mca_btl_tcp_component.tcp_port_min,

mca_btl_tcp_component.tcp_port_min + range));

@@ -624,7 +637,11 @@

return OMPI_ERROR;

}

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

mca_btl_tcp_component.tcp_listen_port = ((struct sockaddr_in*) &inaddr)->sin_port;

mca_btl_tcp_component.tcp_listen_sd = sd;

}

@@ -660,7 +677,11 @@

}

/* register listen port */

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

opal_event_set( &mca_btl_tcp_component.tcp_recv_event,

mca_btl_tcp_component.tcp_listen_sd,

OPAL_EV_READ|OPAL_EV_PERSIST,

@@ -822,6 +843,12 @@

}

/* create a TCP listen socket for incoming connection attempts */

+#if OPAL_WANT_SDP

+ if (mca_btl_tcp_component.sdp_enable) {

+ if(OMPI_SUCCESS != (ret = mca_btl_tcp_component_create_listen(AF_INET_SDP) )) {

+ return 0;

+ }

+ } else {

if(OMPI_SUCCESS != (ret = mca_btl_tcp_component_create_listen(AF_INET) )) {

return 0;

}

@@ -833,7 +860,21 @@

}

#endif

+ }

+#else

+ if(OMPI_SUCCESS != (ret = mca_btl_tcp_component_create_listen(AF_INET) )) {

+ return 0;

+ }

+#if OPAL_WANT_IPV6

+ if((ret = mca_btl_tcp_component_create_listen(AF_INET6)) != OMPI_SUCCESS) {

+ if (!(OMPI_ERR_IN_ERRNO == ret && EAFNOSUPPORT == opal_socket_errno)) {

+ opal_output (0, "mca_btl_tcp_component: IPv6 listening socket failed\n");

+ return 0;

+ }

+#endif

/* publish TCP parameters with the MCA framework */

if(OMPI_SUCCESS != (ret = mca_btl_tcp_component_exchange() )) {

return 0;

Index: ompi/mca/btl/tcp/btl_tcp_endpoint.c

===================================================================

--- ompi/mca/btl/tcp/btl_tcp_endpoint.c (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp_endpoint.c (working copy)

@@ -535,7 +535,14 @@

addrlen = sizeof (struct sockaddr_in6);

}

#endif

+#if OPAL_WANT_SDP

+ if ( mca_btl_tcp_component.sdp_enable){

+ af_family = AF_INET_SDP;

+ addrlen = sizeof(struct sockaddr_in);

+ }

+#endif

btl_endpoint->endpoint_sd = socket(af_family, SOCK_STREAM, 0);

if (btl_endpoint->endpoint_sd < 0) {

btl_endpoint->endpoint_retries++;

Index: ompi/mca/btl/tcp/btl_tcp.h

===================================================================

--- ompi/mca/btl/tcp/btl_tcp.h (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp.h (working copy)

@@ -90,6 +90,9 @@

int tcp_sndbuf; /**< socket sndbuf size */

int tcp_rcvbuf; /**< socket rcvbuf size */

int tcp_disable_family; /**< disabled AF_family */

+#if OPAL_WANT_SDP

+ int sdp_enable; /**< enable SDP */

+#endif /* OPAL_WANT_SDP */

/* free list of fragment descriptors */

ompi_free_list_t tcp_frag_eager;

Index: config/ompi_configure_options.m4

===================================================================

--- config/ompi_configure_options.m4 (revision 17027)

+++ config/ompi_configure_options.m4 (working copy)

@@ -683,6 +683,23 @@

[Enable IPv6 support, but only if the underlying system supports it])

+# Do we want to disable SDP support?

+AC_MSG_CHECKING([if want SDP support])

+AC_ARG_ENABLE([sdp],

+ [AC_HELP_STRING([--enable-sdp],

+ [Enable SDP support (default: disabled)])])

+if test "$enable_sdp" = "yes"; then

+ AC_MSG_RESULT([yes])

+ opal_want_sdp=1

+else

+ AC_MSG_RESULT([no])

+ opal_want_sdp=0

+fi

+AC_DEFINE_UNQUOTED([OPAL_ENABLE_SDP], [$opal_want_sdp],

+ [Enable SDP support])

# Do we want orterun's --prefix behavior to be enabled by default?

AC_MSG_CHECKING([if want orterun "--prefix" behavior to be enabled by default])

Index: orte/mca/oob/tcp/oob_tcp_peer.c

===================================================================

--- orte/mca/oob/tcp/oob_tcp_peer.c (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp_peer.c (working copy)

@@ -371,7 +371,15 @@

opal_net_get_port((struct sockaddr*) &inaddr));

}

- rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+#if OPAL_WANT_SDP

+ if (mca_oob_tcp_component.sdp_enable) {

+ rc = mca_oob_tcp_peer_create_socket(peer, AF_INET_SDP);

+ } else {

+ rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+ }

+#else

+ rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+#endif

if (ORTE_SUCCESS != rc) {

struct timeval tv = { 1,0 };

opal_evtimer_add(&peer->peer_timer_event, &tv);

Index: orte/mca/oob/tcp/oob_tcp.c

===================================================================

--- orte/mca/oob/tcp/oob_tcp.c (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp.c (working copy)

@@ -380,6 +380,13 @@

mca_oob_tcp_component.tcp6_listen_sd = -1;

#endif /* OPAL_WANT_IPV6 */

+#if OPAL_WANT_SDP

+ mca_base_param_reg_int(&mca_oob_tcp_component.super.oob_base,

+ "sdp_enable","Enable SDP for TCP connections",

+ false, false,

+ 0,

+ &mca_oob_tcp_component.sdp_enable);

+#endif

/* initialize state */

mca_oob_tcp_component.tcp_shutdown = false;

mca_oob_tcp_component.tcp_listen_sd = -1;

@@ -514,7 +521,7 @@

int error;

memset(&hints, 0, sizeof(hints));

- hints.ai_family = af_family;

+ hints.ai_family = AF_INET6;

hints.ai_socktype = SOCK_STREAM;

hints.ai_flags = AI_PASSIVE;

@@ -542,9 +549,6 @@

}

#endif /* IPV6_V6ONLY */

#else

- if (AF_INET != af_family) {

- return ORTE_ERROR;

- }

((struct sockaddr_in*) &inaddr)->sin_family = af_family;

((struct sockaddr_in*) &inaddr)->sin_addr.s_addr = INADDR_ANY;

addrlen = sizeof(struct sockaddr_in);

@@ -590,7 +594,11 @@

}

goto socket_binded;

}

- if( AF_INET == af_family ) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

opal_output( 0, "bind() failed: no port available in the range [%d..%d]",

mca_oob_tcp_component.tcp_port_min,

mca_oob_tcp_component.tcp_port_min + range);

@@ -614,7 +622,11 @@

return ORTE_ERROR;

}

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

mca_oob_tcp_component.tcp_listen_port = ((struct sockaddr_in*) &inaddr)->sin_port;

mca_oob_tcp_component.tcp_listen_sd = *target_sd;

}

@@ -647,7 +659,11 @@

}

/* register listen port */

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

opal_event_set(&mca_oob_tcp_component.tcp_recv_event,

*target_sd,

OPAL_EV_READ|OPAL_EV_PERSIST,

@@ -822,6 +838,7 @@

int flags;

/* create a listen socket for incoming connections */

+ /* FIXME add support for SDP */

mca_oob_tcp_component.tcp_listen_sd = socket(AF_INET, SOCK_STREAM, 0);

if(mca_oob_tcp_component.tcp_listen_sd < 0) {

opal_output(0,"mca_oob_tcp_component_init: socket() failed: %s (%d)",

@@ -1194,26 +1211,41 @@

but can't do that since we weren't the HNP. */

mca_oob_tcp_component.tcp_listen_type = OOB_TCP_EVENT;

- rc = mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

- AF_INET);

- if (ORTE_SUCCESS != rc &&

- (EAFNOSUPPORT != opal_socket_errno ||

- mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT)) {

- opal_output(0,

- "mca_oob_tcp_init: unable to create IPv4 listen socket: %s\n",

+#if OPAL_WANT_SDP

+ if ( mca_oob_tcp_component.sdp_enable){

+ rc = mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

+ AF_INET_SDP);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create SDP listen socket: %s\n",

opal_strerror(rc));

- }

+ }

+ } else

+#endif

+ {

+ rc = mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

+ AF_INET);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create IPv4 listen socket: %s\n",

+ opal_strerror(rc));

+ }

#if OPAL_WANT_IPV6

- rc = mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp6_listen_sd,

- AF_INET6);

- if (ORTE_SUCCESS != rc &&

- (EAFNOSUPPORT != opal_socket_errno ||

- mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT)) {

- opal_output(0,

- "mca_oob_tcp_init: unable to create IPv6 listen socket: %s\n",

- opal_strerror(rc));

- }

+ rc = mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp6_listen_sd,

+ AF_INET6);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create IPv6 listen socket: %s\n",

+ opal_strerror(rc));

+ }

#endif

+ }

if (mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_INFO) {

opal_output(0, "%s accepting connections via event library",

ORTE_NAME_PRINT(orte_process_info.my_name));

Index: orte/mca/oob/tcp/oob_tcp.h

===================================================================

--- orte/mca/oob/tcp/oob_tcp.h (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp.h (working copy)

@@ -217,6 +217,9 @@

int tcp6_port_min; /**< Minimum allowed port for the OOB listen socket */

int tcp6_port_range; /**< Range of allowed TCP ports */

#endif /* OPAL_WANT_IPV6 */

+#if OPAL_WANT_SDP

+ int sdp_enable; /**< support for SDP */

+#endif /* OAP_WANT_SDP */

opal_mutex_t tcp_lock; /**< lock for accessing module state */

opal_list_t tcp_events; /**< list of pending events (accepts) */

opal_list_t tcp_msg_post; /**< list of recieves user has posted */

Thanks,

Verkhovsky Lenny.