vtkPStructuredGridConnectivity.h

Go to the documentation of this file.

/*=========================================================================
 
 Program:   Visualization Toolkit
 Module:    vtkPStructuredGridConnectivity.h
 
 Copyright (c) Ken Martin, Will Schroeder, Bill Lorensen
 All rights reserved.
 See Copyright.txt or http://www.kitware.com/Copyright.htm for details.
 
 This software is distributed WITHOUT ANY WARRANTY; without even
 the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
 PURPOSE.  See the above copyright notice for more information.
 
 =========================================================================*/
#ifndef VTKPSTRUCTUREDGRIDCONNECTIVITY_H_
#define VTKPSTRUCTUREDGRIDCONNECTIVITY_H_
 
// VTK include directives
#include "vtkFiltersParallelGeometryModule.h" // For export macro
#include "vtkStructuredGridConnectivity.h"
#include "vtkMPICommunicator.h" // Needed for vtkMPICommunicator::Request
 
// C++ include directives
#include <vector> // For STL vector
 
// Forward declarations
class vtkMultiProcessController;
class vtkMPIController;
class vtkMultiProcessStream;
//class vtkMPICommunicator::Request;
 
class VTKFILTERSPARALLELGEOMETRY_EXPORT vtkPStructuredGridConnectivity :
  public vtkStructuredGridConnectivity
{
public:
  static vtkPStructuredGridConnectivity* New();
  vtkTypeMacro(vtkPStructuredGridConnectivity,vtkStructuredGridConnectivity);
  void PrintSelf(ostream& os, vtkIndent indent );
 
 
  vtkSetMacro( Controller,vtkMultiProcessController* );
  vtkGetMacro( Controller,vtkMultiProcessController* );
 
  void SetNumberOfGrids( const unsigned int N );
 
 
  void RegisterGrid( const int gridID, int extents[6],
      vtkUnsignedCharArray* nodesGhostArray,
      vtkUnsignedCharArray* cellGhostArray,
      vtkPointData* pointData,
      vtkCellData* cellData,
      vtkPoints* gridNodes );
 
 
  int GetNumberOfLocalGrids()
    { return static_cast<int>(this->GridIds.size()); };
 
  int GetGridRank( const int gridID );
 
  bool IsGridRemote( const int gridID );
 
  bool IsGridLocal( const int gridID );
 
  void Initialize();
 
  void ComputeNeighbors();
 
  virtual void CreateGhostLayers( const int N=1 );
 
protected:
  vtkPStructuredGridConnectivity();
  ~vtkPStructuredGridConnectivity();
 
  vtkMultiProcessController *Controller;
  int Rank;
  bool Initialized;
 
  // BTX
  std::vector< int > GridRanks; // Corresponding rank for each grid
  std::vector< int > GridIds;   // List of GridIds, owned by this process
 
  // Data structures to store the remote ghost data of each grid for each one
  // of its neighbors. The first index is the global grid index. The second
  // is the neighbor index.
  std::vector< std::vector< vtkPoints* > >    RemotePoints;
  std::vector< std::vector< vtkPointData* > > RemotePointData;
  std::vector< std::vector< vtkCellData* > >  RemoteCellData;
 
  // Data structures to store the send/receive buffer sizes and corresponding
  // persistent buffers. The first index is the global grid index. The second
  // index is the neighbor index for the given grid.
  std::vector< std::vector< unsigned int > > SendBufferSizes;
  std::vector< std::vector< unsigned int > > RcvBufferSizes;
  std::vector< std::vector< unsigned char* > > SendBuffers;
  std::vector< std::vector< unsigned char* > > RcvBuffers;
 
  int TotalNumberOfSends;
  int TotalNumberOfRcvs;
  int TotalNumberOfMsgs;
 
  // Array of MPI requests
  vtkMPICommunicator::Request *MPIRequests;
  // ETX
 
  bool GridExtentsAreEqual( int rhs[6], int lhs[6] );
 
  bool HasPointData(const int gridIdx);
 
  bool HasCellData(const int gridIdx);
 
  bool HasPoints(const int gridIdx);
 
  void InitializeMessageCounters();
 
  void ClearRemoteData();
 
  void ClearRawBuffers();
 
  void RegisterRemoteGrid( const int gridID, int extents[6], int process );
 
 
  void TransferRemoteNeighborData(
      const int gridIdx,const int nei,const vtkStructuredNeighbor& Neighbor );
 
  virtual void TransferGhostDataFromNeighbors(const int gridID);
 
  void PackGhostData();
 
  void UnpackGhostData();
 
 
  void DeserializeBufferSizesForProcess(
      int *buffersizes, vtkIdType N, const int processId );
 
  void SerializeBufferSizes(int *&sizesbuf, vtkIdType &N);
 
  void ExchangeBufferSizes();
 
  void ExchangeGhostDataInit();
 
  void PostReceives();
 
  void PostSends();
 
  void CommunicateGhostData();
 
  void ExchangeGhostDataPost();
 
  void ExchangeGhostData();
 
 
  void SerializeGhostPoints(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void SerializeDataArray(
      vtkDataArray *dataArray, vtkMultiProcessStream& bytestream );
 
 
  void SerializeFieldData(
      int sourceExtent[6], int targetExtent[6], vtkFieldData *fieldData,
      vtkMultiProcessStream& bytestream );
 
 
  void SerializeGhostPointData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void SerializeGhostCellData(
      const int gridIdx, int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void DeserializeGhostPoints(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void DeserializeDataArray(
      vtkDataArray *&dataArray,const int dataType,
      const int numberOfTuples, const int numberOfComponents,
      vtkMultiProcessStream& bytestream );
 
 
  void DeserializeFieldData(
      int ext[6], vtkFieldData *fieldData,
      vtkMultiProcessStream &bytestream );
 
 
  void DeserializeGhostPointData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void DeserializeGhostCellData(
      const int gridIdx, const int nei,
      int ext[6], vtkMultiProcessStream& bytestream );
 
 
  void SerializeGhostData(
      const int sndGridID, const int rcvGrid, int sndext[6],
      unsigned char*& buffer, unsigned int &size);
 
 
  void DeserializeGhostData(
      const int gridID, const int neiListID,
      const int neiGridIdx, int rcvext[6],
      unsigned char *buffer, unsigned int size );
 
  void ExchangeGridExtents();
 
  void SerializeGridExtents( int *&sndbuffer, vtkIdType &N );
 
 
  void DeserializeGridExtentForProcess(
      int *rcvbuffer, vtkIdType &N, const int processId );
 
private:
  vtkPStructuredGridConnectivity(const vtkPStructuredGridConnectivity& ); // Not implemented
  void operator=(const vtkPStructuredGridConnectivity& ); // Not implemented
};
 
//=============================================================================
//  INLINE METHODS
//=============================================================================
 
 
inline bool vtkPStructuredGridConnectivity::GridExtentsAreEqual(
    int rhs[6], int lhs[6] )
{
  for( int i=0; i < 6; ++i )
    {
    if( rhs[i] != lhs[i] )
      {
      return false;
      }
    }
  return true;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPointData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if( (this->GridPointData[gridIdx] != NULL) &&
      (this->GridPointData[gridIdx]->GetNumberOfArrays() > 0) )
    {
    return true;
    }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasCellData(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if( (this->GridCellData[gridIdx] != NULL) &&
      (this->GridCellData[gridIdx]->GetNumberOfArrays( ) > 0) )
    {
    return true;
    }
  return false;
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::HasPoints(const int gridIdx)
{
  // Sanity check
  assert("pre: grid index is out-of-bounds!" &&
         (gridIdx >= 0) && (gridIdx < static_cast<int>(this->NumberOfGrids)));
 
  if( this->GridPoints[gridIdx] != NULL )
    {
    return true;
    }
  return false;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::InitializeMessageCounters()
{
  this->TotalNumberOfMsgs=this->TotalNumberOfRcvs=this->TotalNumberOfSends=0;
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRawBuffers()
{
  this->SendBufferSizes.clear();
  this->RcvBufferSizes.clear();
 
  // STEP 0: Clear send buffers
  for( unsigned int i=0; i < this->SendBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->SendBuffers[i].size(); ++j )
      {
      delete [] this->SendBuffers[i][j];
      } // END for all neighbors
    this->SendBuffers[i].clear();
    } // END for all grids
  this->SendBuffers.clear();
 
  // STEP 1: Clear rcv buffers
  for( unsigned int i=0; i < this->RcvBuffers.size(); ++i )
    {
    for( unsigned int j=0; j < this->RcvBuffers[i].size(); ++j )
      {
      delete [] this->RcvBuffers[i][j];
      } // END for all neighbors
    this->RcvBuffers[i].clear();
    } // END for all grids
  this->RcvBuffers.clear();
}
 
//------------------------------------------------------------------------------
inline void vtkPStructuredGridConnectivity::ClearRemoteData()
{
  // STEP 0: Clear remote points
  for( unsigned int i=0; i < this->RemotePoints.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePoints[i].size(); ++j )
      {
      if( this->RemotePoints[ i ][ j ] != NULL )
        {
        this->RemotePoints[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePoints[ i ].clear();
    } // END for all i
  this->RemotePoints.clear();
 
  // STEP 1: Clear remote point data
  for( unsigned int i=0; i < this->RemotePointData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemotePointData[i].size(); ++j )
      {
      if( this->RemotePointData[ i ][ j ] != NULL )
        {
        this->RemotePointData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemotePointData[ i ].clear();
    } // END for all i
  this->RemotePointData.clear();
 
  // STEP 2: Clear remote cell data
  for( unsigned int i=0; i < this->RemoteCellData.size(); ++i )
    {
    for( unsigned int j=0; j < this->RemoteCellData[i].size(); ++j )
      {
      if( this->RemoteCellData[ i ][ j ] != NULL )
        {
        this->RemoteCellData[ i ][ j ]->Delete();
        }
      } // END for all j
    this->RemoteCellData[ i ].clear();
    }
  this->RemoteCellData.clear();
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridRemote(const int gridID )
{
  return( !this->IsGridLocal(gridID) );
}
 
//------------------------------------------------------------------------------
inline bool vtkPStructuredGridConnectivity::IsGridLocal(const int gridID)
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID is out-of-bounds" &&
          (gridID >= 0) && (gridID < static_cast<int>(this->NumberOfGrids) ) );
  assert( "pre: GridRanks is not properly allocated" &&
          this->NumberOfGrids == this->GridRanks.size() );
  return( (this->GridRanks[ gridID ] == this->Rank) );
}
 
//------------------------------------------------------------------------------
inline int vtkPStructuredGridConnectivity::GetGridRank( const int gridID )
{
  assert( "pre: Instance has not been intialized!" && this->Initialized );
  assert( "pre: gridID out-of-bounds!" &&
          (gridID >= 0  && gridID < static_cast<int>(this->NumberOfGrids)));
  return( this->GridRanks[ gridID ] );
}
#endif /* VTKPSTRUCTUREDGRIDCONNECTIVITY_H_ */