/* -*- c++ -*- ----------------------------------------------------------
   LAMMPS - Large-scale Atomic/Molecular Massively Parallel Simulator
   https://www.lammps.org/, Sandia National Laboratories
   Steve Plimpton, sjplimp@sandia.gov

   Copyright (2003) Sandia Corporation.  Under the terms of Contract
   DE-AC04-94AL85000 with Sandia Corporation, the U.S. Government retains
   certain rights in this software.  This software is distributed under
   the GNU General Public License.

   See the README file in the top-level LAMMPS directory.
------------------------------------------------------------------------- */

#ifndef LMP_COMM_H
#define LMP_COMM_H

#include "pointers.h"    // IWYU pragma: export

namespace LAMMPS_NS {

class Comm : protected Pointers {
 public:
  int style;     // comm pattern: 0 = 6-way stencil, 1 = irregular tiling
  int layout;    // LAYOUT_UNIFORM = equal-sized bricks
                 // LAYOUT_NONUNIFORM = logical bricks, but diff sizes via LB
                 // LAYOUT_TILED = general tiling, due to RCB LB
  enum { LAYOUT_UNIFORM, LAYOUT_NONUNIFORM, LAYOUT_TILED };
  int mode;    // 0 = single cutoff, 1 = multi-collection cutoff, 2 = multiold-type cutoff
  enum { SINGLE, MULTI, MULTIOLD };

  int me, nprocs;               // proc info
  int ghost_velocity;           // 1 if ghost atoms have velocity, 0 if not
  double cutghost[3];           // cutoffs used for acquiring ghost atoms
  double cutghostuser;          // user-specified ghost cutoff (mode == SINGLE)
  double *cutusermulti;         // per collection user ghost cutoff (mode == MULTI)
  double *cutusermultiold;      // per type user ghost cutoff (mode == MULTIOLD)
  int ncollections;             // # of collections known by comm, used to test if # has changed
  int ncollections_cutoff;      // # of collections stored b cutoff/multi
  int recv_from_partition;      // recv proc layout from this partition
  int send_to_partition;        // send my proc layout to this partition
                                // -1 if no recv or send
  int other_partition_style;    // 0 = recv layout dims must be multiple of
                                //     my layout dims

  int nthreads;    // OpenMP threads per MPI process

  // public settings specific to layout = UNIFORM, NONUNIFORM

  int procgrid[3];                     // procs assigned in each dim of 3d grid
  int user_procgrid[3];                // user request for procs in each dim
  int myloc[3];                        // which proc I am in each dim
  int procneigh[3][2];                 // my 6 neighboring procs, 0/1 = left/right
  double *xsplit, *ysplit, *zsplit;    // fractional (0-1) sub-domain sizes
  int ***grid2proc;                    // which proc owns i,j,k loc in 3d grid

  // public settings specific to layout = TILED

  int rcbnew;              // 1 if just reset by rebalance, else 0
  double mysplit[3][2];    // fractional (0-1) bounds of my sub-domain
  double rcbcutfrac;       // fractional RCB cut by this proc
  int rcbcutdim;           // dimension of RCB cut

  // methods

  Comm(class LAMMPS *);
  virtual ~Comm();
  // NOTE: copy_arrays is called from a constructor and must not be made virtual
  void copy_arrays(class Comm *);
  virtual void init();
  void modify_params(int, char **);

  void set_processors(int, char **);              // set 3d processor grid attributes
  virtual void set_proc_grid(int outflag = 1);    // setup 3d grid of procs

  double get_comm_cutoff();    // determine communication cutoff

  virtual void setup() = 0;                        // setup 3d comm pattern
  virtual void forward_comm(int dummy = 0) = 0;    // forward comm of atom coords
  virtual void reverse_comm() = 0;                 // reverse comm of forces
  virtual void exchange() = 0;                     // move atoms to new procs
  virtual void borders() = 0;                      // setup list of atoms to comm

  // forward/reverse comm from a Pair, Fix, Compute, Dump

  virtual void forward_comm_pair(class Pair *) = 0;
  virtual void reverse_comm_pair(class Pair *) = 0;
  virtual void forward_comm_fix(class Fix *, int size = 0) = 0;
  virtual void reverse_comm_fix(class Fix *, int size = 0) = 0;
  virtual void reverse_comm_fix_variable(class Fix *) = 0;
  virtual void forward_comm_compute(class Compute *) = 0;
  virtual void reverse_comm_compute(class Compute *) = 0;
  virtual void forward_comm_dump(class Dump *) = 0;
  virtual void reverse_comm_dump(class Dump *) = 0;

  // forward comm of an array
  // exchange of info on neigh stencil
  // set processor mapping options

  virtual void forward_comm_array(int, double **) = 0;
  virtual int exchange_variable(int, double *, double *&) = 0;

  // map a point to a processor, based on current decomposition

  virtual void coord2proc_setup() {}
  virtual int coord2proc(double *, int &, int &, int &);

  // partition a global regular grid by proc sub-domains

  void partition_grid(int, int, int, double, int &, int &, int &, int &, int &, int &);

  // memory usage

  virtual double memory_usage() = 0;

  // non-virtual functions common to all Comm styles

  void ring(int, int, void *, int, void (*)(int, char *, void *), void *, void *, int self = 1);
  int rendezvous(int, int, char *, int, int, int *,
                 int (*)(int, char *, int &, int *&, char *&, void *), int, char *&, int, void *,
                 int statflag = 0);

  // extract data useful to other classes

  virtual void *extract(const char *, int &) { return nullptr; }

 protected:
  int bordergroup;    // only communicate this group in borders

  int triclinic;                   // 0 if domain is orthog, 1 if triclinic
  int map_style;                   // non-0 if global->local mapping is done
  int comm_x_only, comm_f_only;    // 1 if only exchange x,f in for/rev comm

  int size_forward;    // # of per-atom datums in forward comm
  int size_reverse;    // # of datums in reverse comm
  int size_border;     // # of datums in forward border comm

  int maxforward, maxreverse;     // max # of datums in forward/reverse comm
  int maxexchange;                // max size of one exchanged atom
  int maxexchange_atom;           // contribution to maxexchange from AtomVec
  int maxexchange_fix;            // static contribution to maxexchange from Fixes
  int maxexchange_fix_dynamic;    // 1 if a fix has a dynamic contribution
  int bufextra;                   // augment send buf size for an exchange atom

  int gridflag;        // option for creating 3d grid
  int mapflag;         // option for mapping procs to 3d grid
  char xyz[4];         // xyz mapping of procs to 3d grid
  char *customfile;    // file with custom proc map
  char *outfile;       // proc grid/map output file

  int otherflag;            // 1 if this partition dependent on another
  int other_style;          // style of dependency
  int other_procgrid[3];    // proc layout of another partition
  int other_coregrid[3];    // core layout of another partition
  int ncores;               // # of cores per node
  int coregrid[3];          // 3d grid of cores within a node
  int user_coregrid[3];     // user request for cores in each dim
  int multi_reduce;         // 1 if multi cutoff is intra-collection cutoff

  void init_exchange();
  int rendezvous_irregular(int, char *, int, int, int *,
                           int (*)(int, char *, int &, int *&, char *&, void *), int, char *&, int,
                           void *, int);
  int rendezvous_all2all(int, char *, int, int, int *,
                         int (*)(int, char *, int &, int *&, char *&, void *), int, char *&, int,
                         void *, int);
  void rendezvous_stats(int, int, int, int, int, int, bigint);

 public:
  enum { MULTIPLE };
};

}    // namespace LAMMPS_NS

#endif

/* ERROR/WARNING messages:

E: Illegal ... command

Self-explanatory.  Check the input script syntax and compare to the
documentation for the command.  You can use -echo screen as a
command-line option when running LAMMPS to see the offending line.

E: Invalid group in comm_modify command

Self-explanatory.

E: Comm_modify group != atom_modify first group

Self-explanatory.

E: Use cutoff/multi keyword to set cutoff in multi mode

Mode is multi so cutoff keyword cannot be used.

E: Invalid cutoff in comm_modify command

Specified cutoff must be >= 0.0.

E: Use cutoff keyword to set cutoff in single mode

Mode is single so cutoff/multi keyword cannot be used.

E: Use cutoff/bytype in mode multi only

Mode is single so cutoff/bytype keyword cannot be used.

E: Cannot set cutoff/multi before simulation box is defined

Self-explanatory.

E: Specified processors != physical processors

The 3d grid of processors defined by the processors command does not
match the number of processors LAMMPS is being run on.

E: Cannot use processors part command without using partitions

See the command-line -partition switch.

E: Invalid partitions in processors part command

Valid partitions are numbered 1 to N and the sender and receiver
cannot be the same partition.

E: Sending partition in processors part command is already a sender

Cannot specify a partition to be a sender twice.

E: Receiving partition in processors part command is already a receiver

Cannot specify a partition to be a receiver twice.

E: Processors grid numa and map style are incompatible

Using numa for gstyle in the processors command requires using
cart for the map option.

E: Processors part option and grid style are incompatible

Cannot use gstyle numa or custom with the part option.

E: Bad grid of processors

The 3d grid of processors defined by the processors command does not
match the number of processors LAMMPS is being run on.

E: Processor count in z must be 1 for 2d simulation

Self-explanatory.

E: Cannot use multi/tiered communication with Newton off

Self-explanatory.

E: Cannot put data on ring from nullptr pointer

W: Communication cutoff is 0.0. No ghost atoms will be generated. Atoms may get lost.

The communication cutoff defaults to the maximum of what is inferred from pair and
bond styles (will be zero, if none are defined) and what is specified via
"comm_modify cutoff" (defaults to 0.0).  If this results to 0.0, no ghost atoms will
be generated and LAMMPS may lose atoms or use incorrect periodic images of atoms in
interaction lists.  To avoid, either define pair style zero with a suitable cutoff
or use comm_modify cutoff.

W: Communication cutoff is shorter than a bond length based estimate. This may lead to errors.

Since LAMMPS stores topology data with individual atoms, all atoms comprising
a bond, angle, dihedral or improper must be present on any sub-domain that
"owns" the atom with the information, either as a local or a ghost atom. The
communication cutoff is what determines up to what distance from a sub-domain
boundary ghost atoms are created.  The communication cutoff is by default the
largest non-bonded cutoff plus the neighbor skin distance, but for short or
non-bonded cutoffs and/or long bonds, this may not be sufficient. This warning
indicates that there is an increased risk of a simulation stopping unexpectedly
because of Bond/Angle/Dihedral/Improper atoms missing.  It can be silenced by
manually setting the communication cutoff via comm_modify cutoff.  However,
since the heuristic used to determine the estimate is not always accurate, it
is not changed automatically and the warning may be ignored depending on the
specific system being simulated.

UNDOCUMENTED

U: OMP_NUM_THREADS environment is not set.

This environment variable must be set appropriately to use the
OPENMP package.

*/
