Open MPI logo

Open MPI Development Mailing List Archives

  |   Home   |   Support   |   FAQ   |   all Development mailing list

Subject: [OMPI devel] SDP support for OPEN-MPI
From: Lenny Verkhovsky (lennyb_at_[hidden])
Date: 2007-12-31 03:41:12


 

Hi,

We would like to add SDP support for OPENMPI.

 

SDP - Socket Direct Protocol is a byte-stream transport protocol
implementing the TCP SOCK_STREAM semantics utilizing transport
offloading capabilities of the InfiniBand fabric

(http://www.mellanox.com/pdf/whitepapers/SDP_Whitepaper.pdf,
http://www.openfabrics.org/archives/aug2005datacenter/das_SDP_Linux.pdf
).

 

SDP can be used to accelerate job start ( oob over sdp ) and IPoIB
performance.

 

The main idea is to use AF_INET_SDP protocol family instead of AF_INET
and AF_INET6 when opening sockets.

SDP will be used in OOB and BTL with appropriate mca parameters -

            -mca btl_tcp_sdp_enable 1

            -mca oob_tcp_sdp_enable 1

 

Since not all functions support this family, the changes were maid only
in critical sections of the code

 

Since SDP support is relevant only for InfiniBand Fabrics you need to
configure sdp support with -enable-sdp flag. SDP will be disabled by
default.

./configure -enable-sdp

 

Test results of running bandwidth and latency of SDP on 2 DDR nodes.

 

BWsize VERBS IPoIB connected IPoIB
datagram btl SDP

1000000 1507.68 665.70
425.21 1272.37

 

LTsize

5 3.82 28.83
28.24 25.73

 

 

 

 

 

Index: opal/include/opal_config_bottom.h

===================================================================

--- opal/include/opal_config_bottom.h (revision 17027)

+++ opal/include/opal_config_bottom.h (working copy)

@@ -509,7 +509,15 @@

 #if !HAVE_DECL_PF_INET6

 #define PF_INET6 PF_UNSPEC

 #endif

+#if !HAVE_DECL_AF_INET_SDP

+#define AF_INET_SDP 27

+#endif

 

+#if OPAL_ENABLE_SDP

+#define OPAL_WANT_SDP 1

+#else

+#define OPAL_WANT_SDP 0

+#endif

 #if defined(__APPLE__) && defined(HAVE_INTTYPES_H)

 /* Prior to Mac OS X 10.3, the length modifier "ll" wasn't

    supported, but "q" was for long long. This isn't ANSI

Index: configure.ac

===================================================================

--- configure.ac (revision 17027)

+++ configure.ac (working copy)

@@ -674,7 +674,7 @@

 #include <netinet/in.h>

 #endif])

 

-AC_CHECK_DECLS([AF_UNSPEC, PF_UNSPEC, AF_INET6, PF_INET6],

+AC_CHECK_DECLS([AF_UNSPEC, PF_UNSPEC, AF_INET6, PF_INET6, AF_INET_SDP],

                [], [], [AC_INCLUDES_DEFAULT

 #if HAVE_SYS_SOCKET_H

 #include <sys/socket.h>

Index: ompi/mca/btl/tcp/btl_tcp_component.c

===================================================================

--- ompi/mca/btl/tcp/btl_tcp_component.c (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp_component.c (working copy)

@@ -263,6 +263,10 @@

 

     mca_btl_tcp_component.tcp_disable_family =

         mca_btl_tcp_param_register_int ("disable_family", NULL, 0);

+#if OPAL_WANT_SDP

+ mca_btl_tcp_component.sdp_enable =

+ mca_btl_tcp_param_register_int("sdp_enable", "Enable SDP for
TCP connections", 0);

+#endif

     return OMPI_SUCCESS;

 }

 

@@ -527,6 +531,11 @@

 

         memset (&hints, 0, sizeof(hints));

         hints.ai_family = af_family;

+#if OPAL_WANT_SDP

+ if ( mca_btl_tcp_component.sdp_enable ) {

+ hints.ai_family = AF_INET6;

+ }

+#endif

         hints.ai_socktype = SOCK_STREAM;

         hints.ai_flags = AI_PASSIVE;

 

@@ -555,7 +564,7 @@

 #endif /* IPV6_V6ONLY */

     }

 #else

- ((struct sockaddr_in*) &inaddr)->sin_family = AF_INET;

+ ((struct sockaddr_in*) &inaddr)->sin_family = af_family;

     ((struct sockaddr_in*) &inaddr)->sin_addr.s_addr = INADDR_ANY;

     addrlen = sizeof(struct sockaddr_in);

 #endif

@@ -600,7 +609,11 @@

             }

             goto socket_binded;

         }

- if( AF_INET == af_family ) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

             BTL_ERROR(("bind() failed: no port available in the range
[%d..%d]",

                        mca_btl_tcp_component.tcp_port_min,

                        mca_btl_tcp_component.tcp_port_min + range));

@@ -624,7 +637,11 @@

         return OMPI_ERROR;

     }

 

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

         mca_btl_tcp_component.tcp_listen_port = ((struct sockaddr_in*)
&inaddr)->sin_port;

         mca_btl_tcp_component.tcp_listen_sd = sd;

     }

@@ -660,7 +677,11 @@

     }

 

     /* register listen port */

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family|| AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

         opal_event_set( &mca_btl_tcp_component.tcp_recv_event,

                         mca_btl_tcp_component.tcp_listen_sd,

                         OPAL_EV_READ|OPAL_EV_PERSIST,

@@ -822,6 +843,12 @@

     }

 

     /* create a TCP listen socket for incoming connection attempts */

+#if OPAL_WANT_SDP

+ if (mca_btl_tcp_component.sdp_enable) {

+ if(OMPI_SUCCESS != (ret =
mca_btl_tcp_component_create_listen(AF_INET_SDP) )) {

+ return 0;

+ }

+ } else {

     if(OMPI_SUCCESS != (ret =
mca_btl_tcp_component_create_listen(AF_INET) )) {

         return 0;

     }

@@ -833,7 +860,21 @@

         }

     }

 #endif

+ }

 

+#else

+ if(OMPI_SUCCESS != (ret =
mca_btl_tcp_component_create_listen(AF_INET) )) {

+ return 0;

+ }

+#if OPAL_WANT_IPV6

+ if((ret = mca_btl_tcp_component_create_listen(AF_INET6)) !=
OMPI_SUCCESS) {

+ if (!(OMPI_ERR_IN_ERRNO == ret && EAFNOSUPPORT ==
opal_socket_errno)) {

+ opal_output (0, "mca_btl_tcp_component: IPv6 listening
socket failed\n");

+ return 0;

+ }

+ }

+#endif

+#endif

     /* publish TCP parameters with the MCA framework */

     if(OMPI_SUCCESS != (ret = mca_btl_tcp_component_exchange() )) {

         return 0;

Index: ompi/mca/btl/tcp/btl_tcp_endpoint.c

===================================================================

--- ompi/mca/btl/tcp/btl_tcp_endpoint.c (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp_endpoint.c (working copy)

@@ -535,7 +535,14 @@

         addrlen = sizeof (struct sockaddr_in6);

     }

 #endif

-

+

+#if OPAL_WANT_SDP

+ if ( mca_btl_tcp_component.sdp_enable){

+ af_family = AF_INET_SDP;

+ addrlen = sizeof(struct sockaddr_in);

+ }

+#endif

+

     btl_endpoint->endpoint_sd = socket(af_family, SOCK_STREAM, 0);

     if (btl_endpoint->endpoint_sd < 0) {

         btl_endpoint->endpoint_retries++;

Index: ompi/mca/btl/tcp/btl_tcp.h

===================================================================

--- ompi/mca/btl/tcp/btl_tcp.h (revision 17027)

+++ ompi/mca/btl/tcp/btl_tcp.h (working copy)

@@ -90,6 +90,9 @@

     int tcp_sndbuf; /**< socket sndbuf size */

     int tcp_rcvbuf; /**< socket rcvbuf size */

     int tcp_disable_family; /**< disabled AF_family */

+#if OPAL_WANT_SDP

+ int sdp_enable; /**< enable SDP */

+#endif /* OPAL_WANT_SDP */

 

     /* free list of fragment descriptors */

     ompi_free_list_t tcp_frag_eager;

Index: config/ompi_configure_options.m4

===================================================================

--- config/ompi_configure_options.m4 (revision 17027)

+++ config/ompi_configure_options.m4 (working copy)

@@ -683,6 +683,23 @@

                    [Enable IPv6 support, but only if the underlying
system supports it])

 

 #

+# Do we want to disable SDP support?

+#

+AC_MSG_CHECKING([if want SDP support])

+AC_ARG_ENABLE([sdp],

+ [AC_HELP_STRING([--enable-sdp],

+ [Enable SDP support (default: disabled)])])

+if test "$enable_sdp" = "yes"; then

+ AC_MSG_RESULT([yes])

+ opal_want_sdp=1

+else

+ AC_MSG_RESULT([no])

+ opal_want_sdp=0

+fi

+AC_DEFINE_UNQUOTED([OPAL_ENABLE_SDP], [$opal_want_sdp],

+ [Enable SDP support])

+

+#

 # Do we want orterun's --prefix behavior to be enabled by default?

 #

 AC_MSG_CHECKING([if want orterun "--prefix" behavior to be enabled by
default])

Index: orte/mca/oob/tcp/oob_tcp_peer.c

===================================================================

--- orte/mca/oob/tcp/oob_tcp_peer.c (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp_peer.c (working copy)

@@ -371,7 +371,15 @@

                         opal_net_get_port((struct sockaddr*) &inaddr));

         }

 

- rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+#if OPAL_WANT_SDP

+ if (mca_oob_tcp_component.sdp_enable) {

+ rc = mca_oob_tcp_peer_create_socket(peer, AF_INET_SDP);

+ } else {

+ rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+ }

+#else

+ rc = mca_oob_tcp_peer_create_socket(peer, inaddr.ss_family);

+#endif

         if (ORTE_SUCCESS != rc) {

             struct timeval tv = { 1,0 };

             opal_evtimer_add(&peer->peer_timer_event, &tv);

Index: orte/mca/oob/tcp/oob_tcp.c

===================================================================

--- orte/mca/oob/tcp/oob_tcp.c (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp.c (working copy)

@@ -380,6 +380,13 @@

     mca_oob_tcp_component.tcp6_listen_sd = -1;

 #endif /* OPAL_WANT_IPV6 */

 

+#if OPAL_WANT_SDP

+ mca_base_param_reg_int(&mca_oob_tcp_component.super.oob_base,

+ "sdp_enable","Enable SDP for TCP
connections",

+ false, false,

+ 0,

+ &mca_oob_tcp_component.sdp_enable);

+#endif

     /* initialize state */

     mca_oob_tcp_component.tcp_shutdown = false;

     mca_oob_tcp_component.tcp_listen_sd = -1;

@@ -514,7 +521,7 @@

         int error;

 

         memset(&hints, 0, sizeof(hints));

- hints.ai_family = af_family;

+ hints.ai_family = AF_INET6;

         hints.ai_socktype = SOCK_STREAM;

         hints.ai_flags = AI_PASSIVE;

         

@@ -542,9 +549,6 @@

     }

 #endif /* IPV6_V6ONLY */

 #else

- if (AF_INET != af_family) {

- return ORTE_ERROR;

- }

     ((struct sockaddr_in*) &inaddr)->sin_family = af_family;

     ((struct sockaddr_in*) &inaddr)->sin_addr.s_addr = INADDR_ANY;

     addrlen = sizeof(struct sockaddr_in);

@@ -590,7 +594,11 @@

             }

             goto socket_binded;

         }

- if( AF_INET == af_family ) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

             opal_output( 0, "bind() failed: no port available in the
range [%d..%d]",

                          mca_oob_tcp_component.tcp_port_min,

                          mca_oob_tcp_component.tcp_port_min + range);

@@ -614,7 +622,11 @@

         return ORTE_ERROR;

     }

 

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

         mca_oob_tcp_component.tcp_listen_port = ((struct sockaddr_in*)
&inaddr)->sin_port;

         mca_oob_tcp_component.tcp_listen_sd = *target_sd;

     }

@@ -647,7 +659,11 @@

     }

 

     /* register listen port */

- if (AF_INET == af_family) {

+#if OPAL_WANT_SDP

+ if( AF_INET == af_family || AF_INET_SDP == af_family) {

+#else

+ if( AF_INET == af_family) {

+#endif

         opal_event_set(&mca_oob_tcp_component.tcp_recv_event,

                        *target_sd,

                        OPAL_EV_READ|OPAL_EV_PERSIST,

@@ -822,6 +838,7 @@

     int flags;

 

     /* create a listen socket for incoming connections */

+ /* FIXME add support for SDP */

     mca_oob_tcp_component.tcp_listen_sd = socket(AF_INET, SOCK_STREAM,
0);

     if(mca_oob_tcp_component.tcp_listen_sd < 0) {

         opal_output(0,"mca_oob_tcp_component_init: socket() failed: %s
(%d)",

@@ -1194,26 +1211,41 @@

            but can't do that since we weren't the HNP. */

         mca_oob_tcp_component.tcp_listen_type = OOB_TCP_EVENT;

 

- rc =
mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

- AF_INET);

- if (ORTE_SUCCESS != rc &&

- (EAFNOSUPPORT != opal_socket_errno ||

- mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT))
{

- opal_output(0,

- "mca_oob_tcp_init: unable to create IPv4 listen
socket: %s\n",

+#if OPAL_WANT_SDP

+ if ( mca_oob_tcp_component.sdp_enable){

+ rc =
mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

+ AF_INET_SDP);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >=
OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create SDP listen
socket: %s\n",

                         opal_strerror(rc));

- }

+ }

+ } else

+#endif

+ {

+ rc =
mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp_listen_sd,

+ AF_INET);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >=
OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create IPv4
listen socket: %s\n",

+ opal_strerror(rc));

+ }

 #if OPAL_WANT_IPV6

- rc =
mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp6_listen_sd,

- AF_INET6);

- if (ORTE_SUCCESS != rc &&

- (EAFNOSUPPORT != opal_socket_errno ||

- mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_CONNECT))
{

- opal_output(0,

- "mca_oob_tcp_init: unable to create IPv6 listen
socket: %s\n",

- opal_strerror(rc));

- }

+ rc =
mca_oob_tcp_create_listen(&mca_oob_tcp_component.tcp6_listen_sd,

+ AF_INET6);

+ if (ORTE_SUCCESS != rc &&

+ (EAFNOSUPPORT != opal_socket_errno ||

+ mca_oob_tcp_component.tcp_debug >=
OOB_TCP_DEBUG_CONNECT)) {

+ opal_output(0,

+ "mca_oob_tcp_init: unable to create IPv6
listen socket: %s\n",

+ opal_strerror(rc));

+ }

 #endif

+ }

         if (mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_INFO) {

             opal_output(0, "%s accepting connections via event
library",

                         ORTE_NAME_PRINT(orte_process_info.my_name));

Index: orte/mca/oob/tcp/oob_tcp.h

===================================================================

--- orte/mca/oob/tcp/oob_tcp.h (revision 17027)

+++ orte/mca/oob/tcp/oob_tcp.h (working copy)

@@ -217,6 +217,9 @@

     int tcp6_port_min; /**< Minimum allowed port
for the OOB listen socket */

     int tcp6_port_range; /**< Range of allowed TCP
ports */

 #endif /* OPAL_WANT_IPV6 */

+#if OPAL_WANT_SDP

+ int sdp_enable; /**< support for SDP */

+#endif /* OAP_WANT_SDP */

     opal_mutex_t tcp_lock; /**< lock for accessing
module state */

     opal_list_t tcp_events; /**< list of pending
events (accepts) */

     opal_list_t tcp_msg_post; /**< list of recieves user
has posted */

 

 

 

Thanks,

Verkhovsky Lenny.