% Prototype Method Send State Machine
% 
% This state machine is targeted towards packet based systems such as VIA.
% Some simplifications may be possible for stream based model like TCP.
% 
% Tasks for this state machine:
% 
% - send message at the head of the queue
% 
%   - message data source exists in two forms:
% 
%     - provided by user - potentially needing to be packed but guaranteed to be
%       available when the CAR is enqueued
% 
%     - provided by associated recv CAR - may not be (completely) available
%       until later (i.e., send could stall)
% 
% - track buffers provided by associated recv CAR
% 
% - provide buffers to associated recv CAR
% 
% - enforce flow control
% 
% 
% This version of the state machine only handles sending from a user provided
% buffer.  It needs to be expanded to include the forwarding cases.

\begin{verbatim}
State [Awaiting CAR],[label="Awaiting\nCAR"]
    Event [CAR Available]
        Invoke_Action([Decide Msg Type],
                      [label="got CAR"])

Action [Decide Msg Type]
    // Note: An otherwise eager message may get converted to a
    // rndv-RTS message if too many outstanding eager messages have already
    // been sent to the receiving process.

    if the type is eager
        // TODO: we may want to convert it to rndv (rts and eventually data)

    Invoke_Action([Check Buffers for Ctrl])

Action [Check Buffers for Ctrl],
       [label="Check Buffers\nfor Ctrl Pkt"]
    // get buffers if we don't already have some
    // always need a buffer for control information
    if buffers are not available
        Change_State([Awaiting Buffers for Ctrl],
                     [label="no bufs"])
    else
        Invoke_Action([Check Flow for Ctrl],
	              [label="bufs ok"])

State [Awaiting Buffers for Ctrl],
      [label="Awaiting Buffers\nfor Ctrl Pkt"]
    Event [Buffers Available]
        Invoke_Action([Check Flow for Ctrl],
                      [label="bufs ok"])

Action [Check Flow for Ctrl],
       [label="Check Flow Control\nfor Ctrl Pkt"]
    if flow restricted // all tensed up
        Change_State([Awaiting Flow for Ctrl],
                     [label="flow\nrestricted"])
    else
        Invoke_Action([Post Send Ctrl],
                      [label="flow ok"])

State [Awaiting Flow for Ctrl],
      [label="Awaiting Flow\nfor Ctrl Pkt"]
    Event [Flow Enabled]
        Invoke_Action([Post Send Ctrl],
                      [label="flow ok"])

Action [Post Send Ctrl],
       [label="Post Send\nfor Ctrl Pkt"]
    type = OD.type
    // pack meta-data into the packet buffer
    // meta-data may include the envelope information (context, src rank,
    // and tag) as well as the ready send bit
    put type in packet buffer
    if type is short or eager
        put the envelope info and rsend bit in the packet buffer
    else if type is rndv-rts message
        put the envelope and LIBA(OD) into the packet buffer
	OD.type = rndv-data
    else if type is rndv-cts
        put the two LIBAs in the packet buffer
    else if type is rndv-data
        put the LIBA in the packet buffer
    else if type is flow control
        // TODO
    
    // pack data from DD into packet buffer (as much as will fit) if we have
    // any data to send
    if type is short, eager, or rndv-data and a DD exists
        if direct access flag is set
            point network send descriptor at the user data (which is pinned)
        else
            pack data into the packet buffer (from the data descriptor)

    post the send to the network device
    if there are no data packets to send (i.e., short)
        Invoke_Action([Signal CAR Completion],
	              [label="msg complete"])
    else if direct access flag is set
        Invoke_Action([Check Flow for Data],
                      [label="more data,\ndirect send"])
    else
        Invoke_Action([Check Buffers for Data],
                      [label="more data,\nbuffered send"])
    
Action [Check Buffers for Data],
       [label="Check Buffers\nfor Data Pkts"]
    // try to get buffers if we need them
    if buffers aren't available
        Change_State([Awaiting Buffers for Data],
		     [label="no bufs"])
    else
        Invoke_Action([Check Flow for Data],
		      [label="bufs ok"])

State [Awaiting Buffers for Data],
      [label="Awaiting Buffers\nfor Data Pkts"]
    Event [Buffers Available]
        Invoke_Action([Check Flow for Data],
                      [label="bufs ok"])

Action [Check Flow for Data],
       [label="Check Flow Control\nfor Data Pkts"]
    if flow restricted
        Change_State([Awaiting Flow for Data],
                     [label="flow\nrestricted"])
    else
        Invoke_Action([Post Send Data],
		      [label="flow ok"])

State [Awaiting Flow for Data],
      [label="Awaiting Flow Control\nfor Data Pkts"]
    Event [Flow Enabled]
        Invoke_Action([Post Send Data],
                      [label="flow ok"])

// TODO: Add buffer packing into flow

Action [Post Send Data],
       [label="Post Send\nfor Data Pkts"]
    if direct access flag is set
        point network send descriptor at the user data (which is pinned)
    else
        pack data into the packet buffer (from the data descriptor)
    post the send to the network device
    // Note: In the implementation, we may want to post multiple packets sends
    // at one time, amortizing the cost of checking for buffers and checking
    // flow control over a set of outgoing packets.  We are uncertain how
    // packing multiple buffers then posting multiple sends will affect
    // performance, but clearly amortizing costs when sending straight out of
    // the user buffer will be a win.

    if there are more data packets to send
        if we are sending directly from the user buffer
            Invoke_Action([Check Flow for Data],
                          [label="more data,\ndirect send"])
        else
            Invoke_Action([Check Buffers for Data],
                          [label="more data,\nbuffered send"])
    else
        Invoke_Action([Signal CAR Completion],
	              [label="msg complete"])
    
Action [Signal CAR Completion], [label="Check/Signal\nCAR Completion"]
    if we did not send all data directly from the user buffer
        reset (local) completion flag
        if we send some of the data directly from the user buffer
            lock CAR.mutex
	        decrement CAR outstanding activities counter
		if outstanding activity counter reaches zero
                    set completion flag
            unlock CAR.mutex
        else // all data sent using packet buffers
            set completion flag
        if completion flag is set
	    decrement completion counter associated with CAR
        
    Invoke_Action([Check Ready CAR])

Action [Check Ready CAR],[label="Check for Ready\nSend CAR"]
    if more CARs are on the send queue
        Invoke_Action([Decide Msg Type],
                      [label="ready CAR"])
    else
        Change_State([Awaiting CAR],
                     [label="no more\nCARs"])
   

\end{verbatim}
