WHAT: Adding communicator add/delete callbacks to MTL.

WHY: MTL will be able to separate messages on different contexts.

WHEN: On trunk (later on v1.5 as well), Tuesday telconf, 5 July 2011

TIMEOUT: Tuesday telconf, 12 July 2011

 The MTL will be allowed to override comm->c_pml_comm member,
since it's unused in pml_cm anyway.

diff -r 0f757f8f3efb -r 8fcf276ee527 ompi/mca/mtl/mtl.h
--- a/ompi/mca/mtl/mtl.h Thu Jun 30 14:13:33 2011 +0300
+++ b/ompi/mca/mtl/mtl.h Fri Jul 01 13:54:20 2011 +0300
@@ -355,6 +355,34 @@
 
 
 /**
+ * Downcall from PML layer when a new communicator is created.
+ *
+ * @param comm  Communicator
+ * @return      OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to initialize/cache a data structure
+ * on the communicator.
+ */
+typedef int (*mca_mtl_base_module_add_comm_fn_t)(
+                          struct mca_mtl_base_module_t* mtl,
+                          struct ompi_communicator_t* comm);
+
+
+/**
+ * Downcall from PML layer when a communicator is destroyed.
+ *
+ * @param comm  Communicator
+ * @return      OMPI_SUCCESS or failure status.
+ *
+ * Provides the MTL the opportunity to cleanup any datastructures
+ * associated with the communicator.
+ */
+typedef int (*mca_mtl_base_module_del_comm_fn_t)(
+                          struct mca_mtl_base_module_t* mtl,
+                          struct ompi_communicator_t* comm);
+
+
+/**
  * MTL module interface functions and attributes.
  */
 struct mca_mtl_base_module_t {
@@ -368,7 +396,6 @@
     mca_mtl_base_module_add_procs_fn_t   mtl_add_procs;
     mca_mtl_base_module_del_procs_fn_t   mtl_del_procs;
     mca_mtl_base_module_finalize_fn_t    mtl_finalize;
-
     mca_mtl_base_module_send_fn_t        mtl_send;
     mca_mtl_base_module_isend_fn_t       mtl_isend;
     mca_mtl_base_module_irecv_fn_t       mtl_irecv;
@@ -376,6 +403,8 @@
 
     /* Optional MTL functions */
     mca_mtl_base_module_cancel_fn_t      mtl_cancel;
+    mca_mtl_base_module_add_comm_fn_t    mtl_add_comm;
+    mca_mtl_base_module_del_comm_fn_t    mtl_del_comm;
 };
 typedef struct mca_mtl_base_module_t mca_mtl_base_module_t;
 
diff -r 0f757f8f3efb -r 8fcf276ee527 ompi/mca/pml/cm/pml_cm.c
--- a/ompi/mca/pml/cm/pml_cm.c Thu Jun 30 14:13:33 2011 +0300
+++ b/ompi/mca/pml/cm/pml_cm.c Fri Jul 01 13:54:20 2011 +0300
@@ -82,14 +82,22 @@
 int
 mca_pml_cm_add_comm(ompi_communicator_t* comm)
 {
+    int ret;
+
     /* should never happen, but it was, so check */
     if (comm->c_contextid > ompi_pml_cm.super.pml_max_contextid) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    /* setup our per-communicator data */
+    /* initialize per-communicator data. MTLs may override this. */
     comm->c_pml_comm = NULL;
 
+    /* notify the MTL about the added communicator */
+    if ((NULL != ompi_mtl->mtl_add_comm) &&
+        (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(add_comm(ompi_mtl, comm))))) {
+        return ret;
+    }
+
     return OMPI_SUCCESS;
 }
 
@@ -97,8 +105,13 @@
 int
 mca_pml_cm_del_comm(ompi_communicator_t* comm)
 {
-    /* clean up our per-communicator data */
-    comm->c_pml_comm = NULL;
+    int ret;
+
+    /* notify the MTL about the deleted communicator */
+    if ((NULL != ompi_mtl->mtl_del_comm) &&
+        (OMPI_SUCCESS != (ret = OMPI_MTL_CALL(del_comm(ompi_mtl, comm))))) {
+        return ret;
+    }
 
     return OMPI_SUCCESS;
 }