PostMaster Class Reference

#include <PostMaster.h>

Collaboration diagram for PostMaster:

Public Member Functions
double *	addToSendBuf (const Eref &e, unsigned int bindIndex, unsigned int size)
	Returns pointer to Send buffer for filling in arguments. More...
double *	addToSetBuf (const Eref &e, unsigned int opIndex, unsigned int size, unsigned int hopType)
	Returns pointer to Set buffer for filling in arguments. More...
void	clearPending ()
	All arrived messages and set calls are handled and cleared. More...
void	clearPendingRecv ()
	Clears arrived messages. More...
void	clearPendingSetGet ()
	Clears arrived set and get calls. More...
void	dispatchSetBuf (const Eref &e)
	Sends off contets of Set buffer. More...
void	finalizeSends ()
	Checks that all sends have gone out. More...
unsigned int	getBufferSize () const
unsigned int	getMyNode () const
unsigned int	getNumNodes () const
void	handleRemoteGet (const Eref &e, const OpFunc *op, int requestingNode)
	Handles 'get' calls from another node, to an object on mynode. More...
void	handleRemoteGetVec (const Eref &e, const OpFunc *op, int requestingNode)
	PostMaster ()
void	process (const Eref &e, ProcPtr p)
void	reinit (const Eref &e, ProcPtr p)
void	remoteFieldGetVec (const Eref &e, unsigned int bindIndex, vector< double > &getRecvBuf)
	This is a blocking call. However, it must still handle other. More...
double *	remoteGet (const Eref &e, unsigned int bindIndex)
	Blocking call to get a value from a remote node. More...
void	remoteGetVec (const Eref &e, unsigned int bindIndex, vector< vector< double > > &getRecvBuf, vector< unsigned int > &size)
	This is a blocking call. However, it must still handle other. More...
void	setBufferSize (unsigned int size)

Static Public Member Functions
static const Cinfo *	initCinfo ()

Static Public Attributes
static const int	CONTROLTAG = 5
static const int	DIETAG = 100
static const int	GETTAG = 3
static const int	MSGTAG = 1
static const unsigned int	reserveBufSize = 1048576
static const int	RETURNTAG = 4
static const unsigned int	setRecvBufSize = 1048576
static const int	SETTAG = 2

Private Attributes
vector< int >	doneIndices_
vector< double >	getHandlerBuf_
int	isSetRecv_
int	isSetSent_
unsigned int	numRecvDone_
vector< vector< double > >	recvBuf_
unsigned int	recvBufSize_
vector< vector< double > >	sendBuf_
vector< unsigned int >	sendSize_
vector< double >	setRecvBuf_
vector< double >	setSendBuf_
int	setSendSize_

Detailed Description

Definition at line 104 of file PostMaster.h.

Constructor & Destructor Documentation

PostMaster::PostMaster

(

)

Definition at line 25 of file PostMaster.cpp.

References getHandlerBuf_, GETTAG, TgtInfo::headerSize, MSGTAG, Shell::myNode(), Shell::numNodes(), recvBuf_, recvBufSize_, reserveBufSize, sendBuf_, setRecvBuf_, setRecvBufSize, and SETTAG.

        :
                recvBufSize_( reserveBufSize ),
                setSendBuf_( setRecvBufSize, 0 ),
                setRecvBuf_( setRecvBufSize, 0 ),
                sendBuf_( Shell::numNodes() ),
                recvBuf_( Shell::numNodes() ),
                sendSize_( Shell::numNodes(), 0 ),
                getHandlerBuf_( TgtInfo::headerSize, 0 ),
                doneIndices_( Shell::numNodes(), 0 ),
                isSetSent_( 1 ), // Flag. Have any pending 'set' gone?
                isSetRecv_( 0 ), // Flag. Has some data come in?
                setSendSize_( 0 ),
                numRecvDone_( 0 )
{
    for ( unsigned int i = 0; i < Shell::numNodes(); ++i ) {
        sendBuf_[i].resize( reserveBufSize, 0 );
    }
#ifdef USE_MPI
    MPI_Barrier( MPI_COMM_WORLD );
    // Post recv for set calls
    MPI_Irecv( &setRecvBuf_[0], setRecvBufSize, MPI_DOUBLE,
                    MPI_ANY_SOURCE,
                    SETTAG, MPI_COMM_WORLD,
                    &setRecvReq_
    );
    // Post recv for get calls.
    MPI_Irecv( &getHandlerBuf_[0], TgtInfo::headerSize, MPI_DOUBLE,
                    MPI_ANY_SOURCE,
                    GETTAG, MPI_COMM_WORLD,
                    &getHandlerReq_
    );
    recvReq_.resize( Shell::numNodes() );
    sendReq_.resize( Shell::numNodes() );
    unsigned int k = 0;
    for ( unsigned int i = 0; i < Shell::numNodes(); ++i ) {
        // Set up the Recv already for later sends. This might be a problem
        // for some polling-based implementations, but let's try for now.
        MPI_Status temp;
        temp.MPI_SOURCE = temp.MPI_TAG = temp.MPI_ERROR = 0;
        doneStatus_.resize( Shell::numNodes(), temp );
        if ( i != Shell::myNode() ) {
            recvBuf_[i].resize( recvBufSize_, 0 );
            MPI_Irecv( &recvBuf_[i][0], recvBufSize_, MPI_DOUBLE,
                i, MSGTAG, MPI_COMM_WORLD,
                &recvReq_[k++]
                // Need to be careful about contiguous indexing for
                // the MPI_request array.
            );
        }
    }
#endif
}

Here is the call graph for this function:

Member Function Documentation

double * PostMaster::addToSendBuf	(	const Eref &	e,
		unsigned int	bindIndex,
		unsigned int	size
	)

Returns pointer to Send buffer for filling in arguments.

Definition at line 442 of file PostMaster.cpp.

References Eref::fieldIndex(), TgtInfo::headerSize, Shell::myNode(), Eref::objId(), recvBufSize_, sendBuf_, sendSize_, and TgtInfo::set().

Referenced by addToBuf().

{
    unsigned int node = e.fieldIndex(); // nasty evil wicked hack
    unsigned int end = sendSize_[node];
    if ( end + TgtInfo::headerSize + size > recvBufSize_ ) {
        // Here we need to activate the fallback second send which will
        // deal with the big block. Also various routines for tracking
        // send size so we don't get too big or small.
        cerr << "Error: PostMaster::addToSendBuf on node " <<
                Shell::myNode() <<
                ": Data size (" << size << ") goes past end of buffer\n";
        assert( 0 );
    }
    TgtInfo* tgt = reinterpret_cast< TgtInfo* >( &sendBuf_[node][end] );
    tgt->set( e.objId(), bindIndex, size );
    end += TgtInfo::headerSize;
    sendSize_[node] = end + size;
    return &sendBuf_[node][end];
}

Here is the call graph for this function:

Here is the caller graph for this function:

double * PostMaster::addToSetBuf	(	const Eref &	e,
		unsigned int	opIndex,
		unsigned int	size,
		unsigned int	hopType
	)

Returns pointer to Set buffer for filling in arguments.

Definition at line 463 of file PostMaster.cpp.

References clearPending(), TgtInfo::headerSize, isSetSent_, Shell::myNode(), Eref::objId(), TgtInfo::set(), setRecvBufSize, setSendBuf_, and setSendSize_.

Referenced by addToBuf().

{
    if ( TgtInfo::headerSize + size > setRecvBufSize ) {
        // Here we need to activate the fallback second send which will
        // deal with the big block. Also various routines for tracking
        // send size so we don't get too big or small.
        cerr << "Error: PostMaster::addToSetBuf on node " <<
                Shell::myNode() <<
                ": Data size (" << size << ") goes past end of buffer\n";
        assert( 0 );
    }
    while ( isSetSent_ == 0 ) { // Can't add a set while old set is pending
        clearPending();
    }
    isSetSent_ = 0;
    TgtInfo* tgt = reinterpret_cast< TgtInfo* >( &setSendBuf_[0] );
    // tgt->set( e.id(), e.dataIndex(), opIndex, e.fieldIndex() );
    tgt->set( e.objId(), opIndex, hopType );
    unsigned int end = TgtInfo::headerSize;
    setSendSize_ = end + size;
    return &setSendBuf_[end];
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::clearPending

(

)

All arrived messages and set calls are handled and cleared.

Definition at line 215 of file PostMaster.cpp.

References clearPendingRecv(), clearPendingSetGet(), and Shell::numNodes().

Referenced by addToSetBuf(), main(), process(), reinit(), remoteFieldGetVec(), remoteGet(), and remoteGetVec().

{
    if ( Shell::numNodes() == 1 )
        return;
    clearPendingSetGet();
    clearPendingRecv();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::clearPendingRecv

(

)

Clears arrived messages.

Definition at line 383 of file PostMaster.cpp.

References TgtInfo::bindIndex(), Element::cinfo(), Eref::dataIndex(), TgtInfo::dataSize(), doneIndices_, Eref::element(), TgtInfo::eref(), Cinfo::getSrcFinfo(), TgtInfo::headerSize, MSGTAG, Shell::myNode(), Shell::numNodes(), numRecvDone_, recvBuf_, recvBufSize_, and SrcFinfo::sendBuffer().

Referenced by clearPending().

{
#ifdef USE_MPI
    int done = 0;
    bool report = false; // for debugging
    MPI_Testsome( Shell::numNodes() -1, &recvReq_[0], &done,
                    &doneIndices_[0], &doneStatus_[0] );
    if ( done == MPI_UNDEFINED )
        return;
    for ( int i = 0; i < done; ++i ) {
        int doneIndex = doneIndices_[i];
        unsigned int recvNode = doneIndex;
        if ( recvNode >= Shell::myNode() )
            recvNode += 1; // Skip myNode
        int recvSize = 0;
        MPI_Get_count( &doneStatus_[i], MPI_DOUBLE, &recvSize );
        int j = 0;
        assert( recvSize <= static_cast< int >( recvBufSize_ ) );
        double* buf = &recvBuf_[ recvNode ][0];
        if ( report ) {
            for ( int j = 0; j < recvSize; j += 4 ) {
                TgtInfo* tgt = reinterpret_cast< TgtInfo * >(buf + j);
                cout << j / 4 << ": " << tgt->eref().dataIndex() << ", " <<
                        buf[j+3] << endl;
            }
        }
        while ( j < recvSize ) {
            const TgtInfo* tgt = reinterpret_cast< const TgtInfo * >( buf );
            const Eref& e = tgt->eref();
            const Finfo *f =
                e.element()->cinfo()->getSrcFinfo( tgt->bindIndex() );
            buf += TgtInfo::headerSize;
            const SrcFinfo* sf = dynamic_cast< const SrcFinfo* >( f );
            assert( sf );
            sf->sendBuffer( e, buf );
            buf += tgt->dataSize();
            j += TgtInfo::headerSize + tgt->dataSize();
            assert( buf - &recvBuf_[recvNode][0] == j );
        }
        // Post the next Irecv.
        unsigned int k = recvNode;
        if ( recvNode > Shell::myNode() )
            k--;
        MPI_Irecv( &recvBuf_[recvNode][0],
                        recvBufSize_, MPI_DOUBLE,
                        recvNode,
                        MSGTAG, MPI_COMM_WORLD,
                        &recvReq_[ k ]
                        // Ensure we have contiguous entries in recvReq_
                 );
    }
    numRecvDone_ += done;
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::clearPendingSetGet

(

)

Clears arrived set and get calls.

Definition at line 299 of file PostMaster.cpp.

References TgtInfo::bindIndex(), TgtInfo::dataSize(), TgtInfo::eref(), handleRemoteGet(), handleRemoteGetVec(), TgtInfo::headerSize, isSetRecv_, isSetSent_, OpFunc::lookop(), MooseGetHop, MooseGetVecHop, MooseSetHop, MooseSetVecHop, op(), OpFunc::opBuffer(), OpFunc::opVecBuffer(), setRecvBuf_, setRecvBufSize, and SETTAG.

Referenced by addToBuf(), and clearPending().

{
#ifdef USE_MPI
    // isSetSent_ is checked before doing another x-node set operation
    // in dispatchSetBuf.
    if ( !isSetSent_ ) {
        MPI_Test( &setSendReq_, &isSetSent_, &setSendStatus_ );
        assert ( isSetSent_ != MPI_UNDEFINED );
    }

    MPI_Test( &setRecvReq_, &isSetRecv_, &setRecvStatus_ );
    if ( isSetRecv_ && isSetRecv_ != MPI_UNDEFINED )
    {
        isSetRecv_ = 0;
        int requestingNode = setRecvStatus_.MPI_SOURCE;
        int count = 0;
        MPI_Get_count( &setRecvStatus_, MPI_DOUBLE, &count );
        // Immediately post another Recv. Needed because we may call
        // the clearPendingSetGet() function recursively. So copy
        // data to another buffer first.
        vector< double > temp( setRecvBuf_.begin(),
                        setRecvBuf_.begin() + count );
        MPI_Irecv( &setRecvBuf_[0], setRecvBufSize, MPI_DOUBLE,
                        MPI_ANY_SOURCE,
                        SETTAG, MPI_COMM_WORLD,
                        &setRecvReq_
        );

        double* buf = &temp[0];
        // Handle arrived Set call
        const TgtInfo* tgt = reinterpret_cast< const TgtInfo * >( buf );
        const Eref& e = tgt->eref();
        const OpFunc *op = OpFunc::lookop( tgt->bindIndex() );
        assert( op );
        if ( tgt->dataSize() == MooseSetHop ) {
            op->opBuffer( e, buf + TgtInfo::headerSize );
        } else if ( tgt->dataSize() == MooseSetVecHop ) {
            op->opVecBuffer( e, buf + TgtInfo::headerSize );
        } else if ( tgt->dataSize() == MooseGetHop ) {
            handleRemoteGet( e, op, requestingNode );
        } else if ( tgt->dataSize() == MooseGetVecHop ) {
            handleRemoteGetVec( e, op, requestingNode );
        }
    }
#endif // USE_MPI
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::dispatchSetBuf

(

const Eref &

e

)

Sends off contets of Set buffer.

Definition at line 487 of file PostMaster.cpp.

References Eref::element(), Eref::getNode(), Element::isGlobal(), Shell::myNode(), Shell::numNodes(), setSendBuf_, setSendSize_, and SETTAG.

Referenced by dispatchBuffers().

{
    assert ( e.element()->isGlobal() || e.getNode() != Shell::myNode() );
#ifdef USE_MPI
    if ( e.element()->isGlobal() ) {
        for ( unsigned int i = 0; i < Shell::numNodes(); ++i ) {
            if ( i != Shell::myNode() ) {
            // A bcast would be marginally more efficient, but would need
            // us to inform all target nodes to expect one. So just do
            // multiple sends.
                MPI_Isend(
                    &setSendBuf_[0], setSendSize_, MPI_DOUBLE,
                    i,      // Where to send to.
                    SETTAG, MPI_COMM_WORLD,
                    &setSendReq_
        // Need to monitor all the sends to make sure they all complete
        // before permitting another 'set'
                );
            }
        }
    } else {
        MPI_Isend(
            &setSendBuf_[0], setSendSize_, MPI_DOUBLE,
                e.getNode(),        // Where to send to.
                SETTAG, MPI_COMM_WORLD,
                &setSendReq_
        );
    }
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::finalizeSends

(

)

Checks that all sends have gone out.

Finalize all outgoing messages, clear the sendSize vector so that we can handle another round of messages.

Definition at line 147 of file PostMaster.cpp.

References Shell::numNodes(), and sendSize_.

Referenced by process(), and reinit().

{
#ifdef USE_MPI
    static vector< MPI_Status > status( Shell::numNodes() );
    MPI_Waitall( Shell::numNodes() -1, &sendReq_[0], &status[0] );
    for (unsigned int i = 0; i < Shell::numNodes(); ++i ) {
        sendSize_[i] = 0;
    }
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

unsigned int PostMaster::getBufferSize

(

)

const

Definition at line 714 of file PostMaster.cpp.

References sendBuf_.

Referenced by initCinfo().

{
    if ( sendBuf_.size() == 0 )
        return 0;

    return sendBuf_[0].size();
}

Here is the caller graph for this function:

unsigned int PostMaster::getMyNode

(

)

const

Definition at line 709 of file PostMaster.cpp.

References Shell::myNode().

Referenced by initCinfo().

{
    return Shell::myNode();
}

Here is the call graph for this function:

Here is the caller graph for this function:

unsigned int PostMaster::getNumNodes

(

)

const

Definition at line 704 of file PostMaster.cpp.

References Shell::numNodes().

Referenced by initCinfo().

{
    return Shell::numNodes();
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::handleRemoteGet	(	const Eref &	e,
		const OpFunc *	op,
		int	requestingNode
	)

Handles 'get' calls from another node, to an object on mynode.

Definition at line 223 of file PostMaster.cpp.

References OpFunc::opBuffer(), reserveBufSize, and RETURNTAG.

Referenced by clearPendingSetGet().

{
#ifdef USE_MPI
    static double getReturnBuf[reserveBufSize];
    op->opBuffer( e, &getReturnBuf[0] ); // stuff return value into buf.
    // Send out the data. Blocking. Don't want any other gets till done
    int size = getReturnBuf[0];
    MPI_Send(
        &getReturnBuf[1], size, MPI_DOUBLE,
        requestingNode,         // Where to send to.
        RETURNTAG, MPI_COMM_WORLD
    );
#endif // USE_MPI
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::handleRemoteGetVec	(	const Eref &	e,
		const OpFunc *	op,
		int	requestingNode
	)

Definition at line 284 of file PostMaster.cpp.

References innerGetVec(), reserveBufSize, and RETURNTAG.

Referenced by clearPendingSetGet().

{
#ifdef USE_MPI
    static double getReturnBuf[reserveBufSize];
    int k = innerGetVec( e, op, getReturnBuf );
    // Send out the data. Blocking. Don't want any other gets till done
    MPI_Send(
        &getReturnBuf[0], k, MPI_DOUBLE,
        requestingNode,         // Where to send to.
        RETURNTAG, MPI_COMM_WORLD
    );
#endif // USE_MPI
}

Here is the call graph for this function:

Here is the caller graph for this function:

const Cinfo * PostMaster::initCinfo ( )

static

Definition at line 82 of file PostMaster.cpp.

References getBufferSize(), getMyNode(), getNumNodes(), Neutral::initCinfo(), myNode, numNodes, process(), reinit(), and setBufferSize().

Referenced by init().

{
        // Field Definitions
        static ReadOnlyValueFinfo< PostMaster, unsigned int > numNodes(
            "numNodes",
            "Returns number of nodes that simulation runs on.",
            &PostMaster::getNumNodes
        );
        static ReadOnlyValueFinfo< PostMaster, unsigned int > myNode(
            "myNode",
            "Returns index of current node.",
            &PostMaster::getMyNode
        );
        static ValueFinfo< PostMaster, unsigned int > bufferSize(
            "bufferSize",
            "Size of the send a receive buffers for each node.",
            &PostMaster::setBufferSize,
            &PostMaster::getBufferSize
        );
        // MsgDest Definitions
        static DestFinfo process( "process",
            "Handles process call",
            new EpFunc1< PostMaster, ProcPtr >( &PostMaster::process ) );
        static DestFinfo reinit( "reinit",
            "Handles reinit call",
            new EpFunc1< PostMaster, ProcPtr >( &PostMaster::reinit ) );

        // SharedFinfo Definitions
        static Finfo* procShared[] = {
            &process, &reinit
        };
        static SharedFinfo proc( "proc",
            "Shared message for process and reinit",
            procShared, sizeof( procShared ) / sizeof( const Finfo* )
        );

    static Finfo* postMasterFinfos[] = {
        &numNodes,  // ReadOnlyValue
        &myNode,    // ReadOnlyValue
        &bufferSize,    // ReadOnlyValue
        &proc       // SharedFinfo
    };

    static Dinfo< PostMaster > dinfo;
    static Cinfo postMasterCinfo (
        "PostMaster",
        Neutral::initCinfo(),
        postMasterFinfos,
        sizeof( postMasterFinfos ) / sizeof ( Finfo* ),
        &dinfo
    );

    return &postMasterCinfo;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::process	(	const Eref &	e,
		ProcPtr	p
	)

Definition at line 190 of file PostMaster.cpp.

References clearPending(), finalizeSends(), MSGTAG, Shell::myNode(), Shell::numNodes(), numRecvDone_, sendBuf_, and sendSize_.

Referenced by initCinfo().

{
#ifdef USE_MPI
    unsigned int reqIndex = 0;
    for ( unsigned int i = 0; i < Shell::numNodes(); ++i )
    {
        if ( i == Shell::myNode() ) continue;
        // MPI_scatter would have been better but it doesn't allow
        // one to post larger recvs than the actual data sent.
        MPI_Isend(
            &sendBuf_[i][0], sendSize_[i], MPI_DOUBLE,
            i,      // Where to send to.
            MSGTAG, MPI_COMM_WORLD,
            &sendReq_[ reqIndex++ ]
        );
        clearPending(); // Try to interleave communications.
    }
    while ( numRecvDone_ < Shell::numNodes() -1 )
        clearPending();
    finalizeSends();
    MPI_Barrier( MPI_COMM_WORLD );
    numRecvDone_ = 0;
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::reinit	(	const Eref &	e,
		ProcPtr	p
	)

PostMaster class: handles cross-node messaging using MPI. Identical to the Process call: sends out what needs to go, and then waits for any incoming messages and passes them on.

Definition at line 164 of file PostMaster.cpp.

References clearPending(), finalizeSends(), MSGTAG, Shell::myNode(), Shell::numNodes(), numRecvDone_, sendBuf_, and sendSize_.

Referenced by initCinfo().

{
#ifdef USE_MPI
    // MPI_Barrier( MPI_COMM_WORLD );
    unsigned int reqIndex = 0;
    for ( unsigned int i = 0; i < Shell::numNodes(); ++i )
    {
        if ( i == Shell::myNode() ) continue;
        // MPI_scatter would have been better but it doesn't allow
        // one to post larger recvs than the actual data sent.
        MPI_Isend(
            &sendBuf_[i][0], sendSize_[i], MPI_DOUBLE,
            i,      // Where to send to.
            MSGTAG, MPI_COMM_WORLD,
            &sendReq_[ reqIndex++ ]
        );
        clearPending(); // Try to interleave communications.
    }
    while ( numRecvDone_ < Shell::numNodes() -1 )
        clearPending();
    finalizeSends();
    MPI_Barrier( MPI_COMM_WORLD );
    numRecvDone_ = 0;
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::remoteFieldGetVec	(	const Eref &	e,
		unsigned int	bindIndex,
		vector< double > &	getRecvBuf
	)

This is a blocking call. However, it must still handle other.

Definition at line 568 of file PostMaster.cpp.

References clearPending(), Eref::element(), Eref::getNode(), TgtInfo::headerSize, Element::isGlobal(), isSetSent_, MooseGetVecHop, Shell::myNode(), Eref::objId(), reserveBufSize, RETURNTAG, TgtInfo::set(), and SETTAG.

Referenced by remoteFieldGetVec().

{
#ifdef USE_MPI
    static double getSendBuf[TgtInfo::headerSize];
    static MPI_Request getSendReq;
    static MPI_Request getRecvReq;
    static MPI_Status doneStatus;
#endif
    unsigned int targetNode = e.getNode();
    assert( targetNode != Shell::myNode() );
    getRecvBuf.clear();
    getRecvBuf.resize( reserveBufSize );

#ifdef USE_MPI
    while ( isSetSent_ == 0 ) {
            // Can't request a 'get' while old set is
            // pending, lest the 'get' depend on the 'set'.
        clearPending();
    }
    TgtInfo* tgt = reinterpret_cast< TgtInfo* >( &getSendBuf[0] );
    tgt->set( e.objId(), bindIndex, MooseGetVecHop );
    assert ( !e.element()->isGlobal() );

    // Post receive for return value.
    MPI_Irecv(  &getRecvBuf[0],
                reserveBufSize, MPI_DOUBLE,
                targetNode,
                RETURNTAG, MPI_COMM_WORLD,
                &getRecvReq
            );
    // Now post send to request the data
    MPI_Isend(
                &getSendBuf[0], TgtInfo::headerSize, MPI_DOUBLE,
                targetNode,             // Where to send to.
                SETTAG, MPI_COMM_WORLD,
                &getSendReq
            );
    // Poll till the value comes back. We don't bother to
    // check what happened with the send.
    // While polling be sure to handle any other requests to avoid deadlock
    int done = 0;
    while( !done ) {
        MPI_Test( &getRecvReq, &done, &doneStatus );
        assert ( done != MPI_UNDEFINED );
        clearPending();
    }
    // Now we have the data back.
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

double * PostMaster::remoteGet	(	const Eref &	e,
		unsigned int	bindIndex
	)

Blocking call to get a value from a remote node.

This is a blocking call. However, it must not block other requests.

Definition at line 520 of file PostMaster.cpp.

References clearPending(), Eref::element(), Eref::getNode(), TgtInfo::headerSize, Element::isGlobal(), isSetSent_, MooseGetHop, Shell::myNode(), Eref::objId(), RETURNTAG, TgtInfo::set(), setRecvBufSize, and SETTAG.

Referenced by remoteGet().

{
    static double getRecvBuf[setRecvBufSize];
#ifdef USE_MPI
    static double getSendBuf[TgtInfo::headerSize];
    static MPI_Request getSendReq;
    static MPI_Request getRecvReq;
    static MPI_Status getSendStatus;

    while ( isSetSent_ == 0 ) {
            // Can't request a 'get' while old set is
            // pending, lest the 'get' depend on the 'set'.
        clearPending();
    }
    TgtInfo* tgt = reinterpret_cast< TgtInfo* >( &getSendBuf[0] );
    tgt->set( e.objId(), bindIndex, MooseGetHop );
    assert ( !e.element()->isGlobal() && e.getNode() != Shell::myNode() );
    // Post receive for return value.
    MPI_Irecv(      &getRecvBuf[0],
                    setRecvBufSize, MPI_DOUBLE,
                    e.getNode(),
                    RETURNTAG, MPI_COMM_WORLD,
                    &getRecvReq
             );
    // Now post send to request the data
    MPI_Isend(
        &getSendBuf[0], TgtInfo::headerSize, MPI_DOUBLE,
            e.getNode(),        // Where to send to.
            SETTAG, MPI_COMM_WORLD,
            &getSendReq
    );
    int complete = 0;
    // Poll till the value comes back. We don't bother to
    // check what happened with the send.
    // While polling be sure to handle any other requests to avoid deadlock
    while ( !complete ) {
        MPI_Test( &getRecvReq, &complete, &getSendStatus );
        assert ( complete != MPI_UNDEFINED );
        clearPending();
    }
#endif
    return &getRecvBuf[0];
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::remoteGetVec	(	const Eref &	e,
		unsigned int	bindIndex,
		vector< vector< double > > &	getRecvBuf,
		vector< unsigned int > &	size
	)

This is a blocking call. However, it must still handle other.

Definition at line 623 of file PostMaster.cpp.

References clearPending(), Eref::element(), TgtInfo::headerSize, Element::isGlobal(), isSetSent_, MooseGetVecHop, Shell::myNode(), Shell::numNodes(), Eref::objId(), reserveBufSize, RETURNTAG, TgtInfo::set(), and SETTAG.

Referenced by remoteGetVec().

{
#ifdef USE_MPI
    static double getSendBuf[TgtInfo::headerSize];
    static vector< MPI_Request > getSendReq( Shell::numNodes() );
    static vector< MPI_Request > getRecvReq( Shell::numNodes() );
    static vector< MPI_Status > doneStatus( Shell::numNodes() );
#endif
    static vector< double > temp( reserveBufSize, 0 );
    numOnNode.clear();
    numOnNode.resize( Shell::numNodes(), 0 );
    getRecvBuf.clear();
    getRecvBuf.resize( Shell::numNodes(), temp );

#ifdef USE_MPI
    while ( isSetSent_ == 0 ) {
            // Can't request a 'get' while old set is
            // pending, lest the 'get' depend on the 'set'.
        clearPending();
    }
    TgtInfo* tgt = reinterpret_cast< TgtInfo* >( &getSendBuf[0] );
    tgt->set( e.objId(), bindIndex, MooseGetVecHop );
    assert ( !e.element()->isGlobal() );

    unsigned int k = 0;
    for ( unsigned int i = 0; i < Shell::numNodes(); ++i ) {
        if ( i != Shell::myNode() ) {
            // Post receive for return value.
            MPI_Irecv(  &getRecvBuf[i][0],
                    reserveBufSize, MPI_DOUBLE,
                    i,
                    RETURNTAG, MPI_COMM_WORLD,
                    &getRecvReq[k]
            );
            // Now post send to request the data
            MPI_Isend(
                &getSendBuf[0], TgtInfo::headerSize, MPI_DOUBLE,
                i,          // Where to send to.
                SETTAG, MPI_COMM_WORLD,
                &getSendReq[k]
            );
            k++;
        }
    }
    // Poll till the value comes back. We don't bother to
    // check what happened with the send.
    // While polling be sure to handle any other requests to avoid deadlock
    int done = 0;
    unsigned int received = 0;
    vector< int > getDoneIndices( Shell::numNodes(), 0 );
    while( received < Shell::numNodes() - 1 ) {
        MPI_Testsome( Shell::numNodes() -1, &getRecvReq[0], &done,
                    &getDoneIndices[0], &doneStatus[0] );
        if ( done == MPI_UNDEFINED )
            continue;
        received += done;
        for ( int i = 0; i < done; ++i ) {
            int doneIndex = getDoneIndices[i];
            unsigned int recvNode = doneIndex;
            if ( recvNode >= Shell::myNode() )
                recvNode += 1; // Skip myNode
            /*
            int recvSize = 0;
            MPI_Get_count( &doneStatus[doneIndex],
                            MPI_DOUBLE, &recvSize );
            size[recvNode] = recvSize;
                            */
            numOnNode[recvNode] = getRecvBuf[recvNode][0];
        }
        clearPending();
    }
    // Now we have the whole mess back.
#endif
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PostMaster::setBufferSize

(

unsigned int

size

)

Definition at line 722 of file PostMaster.cpp.

References resize(), and sendBuf_.

Referenced by initCinfo().

{
    for ( unsigned int i =0; i < sendBuf_.size(); ++i )
        sendBuf_[i].resize( size );
}

Here is the call graph for this function:

Here is the caller graph for this function:

Member Data Documentation

const int PostMaster::CONTROLTAG = 5

static

Definition at line 153 of file PostMaster.h.

const int PostMaster::DIETAG = 100

static

Definition at line 154 of file PostMaster.h.

vector< int > PostMaster::doneIndices_

private

Definition at line 175 of file PostMaster.h.

Referenced by clearPendingRecv().

vector< double > PostMaster::getHandlerBuf_

private

Definition at line 164 of file PostMaster.h.

Referenced by PostMaster().

const int PostMaster::GETTAG = 3

static

Definition at line 151 of file PostMaster.h.

Referenced by PostMaster().

int PostMaster::isSetRecv_

private

Definition at line 177 of file PostMaster.h.

Referenced by clearPendingSetGet().

int PostMaster::isSetSent_

private

Definition at line 176 of file PostMaster.h.

Referenced by addToSetBuf(), clearPendingSetGet(), remoteFieldGetVec(), remoteGet(), and remoteGetVec().

const int PostMaster::MSGTAG = 1

static

Definition at line 149 of file PostMaster.h.

Referenced by clearPendingRecv(), PostMaster(), process(), and reinit().

unsigned int PostMaster::numRecvDone_

private

Definition at line 179 of file PostMaster.h.

Referenced by clearPendingRecv(), process(), and reinit().

vector< vector< double > > PostMaster::recvBuf_

private

Definition at line 162 of file PostMaster.h.

Referenced by clearPendingRecv(), and PostMaster().

unsigned int PostMaster::recvBufSize_

private

Definition at line 157 of file PostMaster.h.

Referenced by addToSendBuf(), clearPendingRecv(), and PostMaster().

const unsigned int PostMaster::reserveBufSize = 1048576

static

Definition at line 147 of file PostMaster.h.

Referenced by handleRemoteGet(), handleRemoteGetVec(), innerGetVec(), PostMaster(), remoteFieldGetVec(), and remoteGetVec().

const int PostMaster::RETURNTAG = 4

static

Definition at line 152 of file PostMaster.h.

Referenced by handleRemoteGet(), handleRemoteGetVec(), remoteFieldGetVec(), remoteGet(), and remoteGetVec().

vector< vector< double > > PostMaster::sendBuf_

private

Definition at line 161 of file PostMaster.h.

Referenced by addToSendBuf(), getBufferSize(), PostMaster(), process(), reinit(), and setBufferSize().

vector< unsigned int > PostMaster::sendSize_

private

Definition at line 163 of file PostMaster.h.

Referenced by addToSendBuf(), finalizeSends(), process(), and reinit().

vector< double > PostMaster::setRecvBuf_

private

Definition at line 160 of file PostMaster.h.

Referenced by clearPendingSetGet(), and PostMaster().

const unsigned int PostMaster::setRecvBufSize = 1048576

static

Definition at line 148 of file PostMaster.h.

Referenced by addToSetBuf(), clearPendingSetGet(), PostMaster(), and remoteGet().

vector< double > PostMaster::setSendBuf_

private

Definition at line 159 of file PostMaster.h.

Referenced by addToSetBuf(), and dispatchSetBuf().

int PostMaster::setSendSize_

private

Definition at line 178 of file PostMaster.h.

Referenced by addToSetBuf(), and dispatchSetBuf().

const int PostMaster::SETTAG = 2

static

Definition at line 150 of file PostMaster.h.

Referenced by clearPendingSetGet(), dispatchSetBuf(), PostMaster(), remoteFieldGetVec(), remoteGet(), and remoteGetVec().

---

The documentation for this class was generated from the following files:

• moose-core/mpi/PostMaster.h
• moose-core/mpi/PostMaster.cpp