int topodistNodeCommInfo(topodist_t *td, MPI_Comm comm, int *partnerOnNode, int *partnerOffNode, MPI_Comm *nodecomm_p, int *totalOffNode)
nOffNode = 0; nOnNode = 0;
/* For each partner, compute on/offnode information */ for (i=0; i<td->ntargets; i++) { /* are the topoinfo values comparable? */ sameNode = 0; if (td->tarray[i].nodeidx == td->myarray.nodeidx && td->tarray[i].nlevels == td->myarray.nlevels) { /* Check the topo indices above the node */ sameNode = 1; for (j=td->myarray.nodeidx; j<td->myarray.nlevels; j++) { if (td->tarray[i].mycoords[j] != td->myarray.mycoords[j]) sameNode = 0; } } if (sameNode) { nOnNode++; } else { nOffNode++; } }
*partnerOnNode = nOnNode; *partnerOffNode = nOffNode;
/* Find all of information by node, as defined above */ MPI_Comm_rank(comm, &rank); color = 0; /* Check for the case where no interconnect network is known */ if (td->myarray.nodeidx < td->myarray.nlevels) color = td->myarray.mycoords[td->myarray.nodeidx]; for (i=td->myarray.nodeidx+1; i<td->myarray.nlevels; i++) color = td->myarray.mycoords[i] + td->myarray.maxcoords[i-1]*color; MPI_Comm_split(comm, color, rank, &nodecomm);
anyoff = (nOffNode > 0); MPI_Allreduce(&anyoff, totalOffNode, 1, MPI_INT, MPI_SUM, nodecomm); MPI_Comm_rank(nodecomm, &nrank); *nodecomm_p = nodecomm;
return 0; }
#include <string.h> int topodistPrint(topodist_t *td, FILE *fp) { int nOffNode, nOnNode, totalOffNode, nodeOnNode, nodeOffNode, nrank, nsize, rank, size, err; MPI_Comm nodecomm;
MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size);
if (rank == 0) { printf("Test of on/off node determination\n"); fflush(stdout); }
err = topodistNodeCommInfo(td, MPI_COMM_WORLD, &nOnNode, &nOffNode, &nodecomm, &totalOffNode);
MPI_Allreduce(&nOnNode, &nodeOnNode, 1, MPI_INT, MPI_SUM, nodecomm); MPI_Allreduce(&nOffNode, &nodeOffNode, 1, MPI_INT, MPI_SUM, nodecomm);
MPI_Comm_rank(nodecomm, &nrank); MPI_Comm_size(nodecomm, &nsize); if (nrank == 0) { printf("rank = %d, nsize = %d, Sends: Node On = %d, Off = %d; procs sending off = %d\n", rank, nsize, nodeOnNode, nodeOffNode, totalOffNode); fflush(stdout); } MPI_Comm_free(&nodecomm); MPI_Barrier(MPI_COMM_WORLD);
{ char coordStr[1024]; int j; coordStr[0] = 0; for (j=td->myarray.nodeidx; j<td->myarray.nlevels; j++) { char numstr[20]; snprintf(numstr, 20, "%d", td->myarray.mycoords[j]); strncat(coordStr, numstr, 1024); strncat(coordStr, ",", 1024); } if (rank > 0) { MPI_Recv((void*)0, 0, MPI_INT, rank-1, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE); } printf("%d:[%s] OnNode = %d, OffNode = %d\n", rank, coordStr, nOnNode, nOffNode); fflush(stdout); if (rank < size-1) { MPI_Send((void*)0, 0, MPI_INT, rank+1, 1, MPI_COMM_WORLD); } } return 0; }
#include <math.h> void topodistTest(FILE *fp) { topodist_t *dt; topoinfo_t *ti; int rank, size, sends[4], m, ns; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); m = sqrt((double)size); ns = 0; if (rank > 0) sends[ns++] = rank - 1; if (rank < size-1) sends[ns++] = rank + 1; if (rank - m >= 0) sends[ns++] = rank - m; if (rank + m <= size-1) sends[ns++] = rank + m; topoInit(1,&ti); topodistInit(MPI_COMM_WORLD, ns, sends, ns, sends, ti, &dt); topodistPrint(dt, fp); topodistFree(dt); }