/*
 * (C) Copyright 2013 ECMWF.
 *
 * This software is licensed under the terms of the Apache Licence Version 2.0
 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 * In applying this licence, ECMWF does not waive the privileges and immunities
 * granted to it by virtue of its status as an intergovernmental organisation
 * nor does it submit to any jurisdiction.
 */

#include <algorithm>
#include <cmath>
#include <cstdarg>
#include <functional>
#include <limits>

#include "atlas/array.h"
#include "atlas/field/Field.h"
#include "atlas/functionspace/NodeColumns.h"
#include "atlas/library/config.h"
#include "atlas/mesh/IsGhostNode.h"
#include "atlas/mesh/Nodes.h"
#include "atlas/parallel/GatherScatter.h"
#include "atlas/parallel/omp/omp.h"
#include "atlas/runtime/Exception.h"
#include "atlas/runtime/Trace.h"


#undef atlas_omp_critical_ordered
#define atlas_omp_critical_ordered atlas_omp_critical

namespace atlas {
namespace functionspace {
namespace detail {

namespace {

template <typename T, typename Field>
array::LocalView<T, 3> make_leveled_view( Field& field ) {
    using namespace array;
    if ( field.levels() ) {
        if ( field.variables() ) {
            return make_view<T, 3>( field ).slice( Range::all(), Range::all(), Range::all() );
        }
        else {
            return make_view<T, 2>( field ).slice( Range::all(), Range::all(), Range::dummy() );
        }
    }
    else {
        if ( field.variables() ) {
            return make_view<T, 2>( field ).slice( Range::all(), Range::dummy(), Range::all() );
        }
        else {
            return make_view<T, 1>( field ).slice( Range::all(), Range::dummy(), Range::dummy() );
        }
    }
}

template <typename T, typename Field>
array::LocalView<T, 2> make_leveled_scalar_view( Field& field ) {
    using namespace array;
    if ( field.levels() ) {
        return make_view<T, 2>( field ).slice( Range::all(), Range::all() );
    }
    else {
        return make_view<T, 1>( field ).slice( Range::all(), Range::dummy() );
    }
}

template <typename T, typename Field>
array::LocalView<T, 2> make_surface_view( Field& field ) {
    using namespace array;
    if ( field.variables() ) {
        return make_view<T, 2>( field ).slice( Range::all(), Range::all() );
    }
    else {
        return make_view<T, 1>( field ).slice( Range::all(), Range::dummy() );
    }
}

template <typename T, typename Field>
array::LocalView<T, 2> make_per_level_view( Field& field ) {
    using namespace array;
    if ( field.rank() == 2 ) {
        return make_view<T, 2>( field ).slice( Range::all(), Range::all() );
    }
    else {
        return make_view<T, 1>( field ).slice( Range::all(), Range::dummy() );
    }
}

}  // namespace

namespace {
inline double sqr( const double& val ) {
    return val * val;
}
}  // namespace

namespace detail {  // Collectives implementation

template <typename T>
void dispatch_sum( const NodeColumns& fs, const Field& field, T& result, idx_t& N ) {
    const mesh::IsGhostNode is_ghost( fs.nodes() );
    const array::LocalView<const T, 2> arr = make_leveled_scalar_view<const T>( field );
    T local_sum                            = 0;
    const idx_t npts                       = std::min<idx_t>( arr.shape( 0 ), fs.nb_nodes() );
    const idx_t nlev                       = arr.shape( 1 );
  atlas_omp_pragma( omp parallel for default(shared) reduction(+:local_sum) )
  for( idx_t n=0; n<npts; ++n ) {
      if ( !is_ghost( n ) ) {
          for ( idx_t l = 0; l < nlev; ++l ) {
              local_sum += arr( n, l );
          }
      }
  }
  ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( local_sum, result, eckit::mpi::sum() ); }

  N = fs.nb_nodes_global() * arr.shape( 1 );
}

template <typename T>
void sum( const NodeColumns& fs, const Field& field, T& result, idx_t& N ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_sum( fs, field, result, N );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                int tmp;
                dispatch_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_INT64: {
                long tmp;
                dispatch_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_REAL32: {
                float tmp;
                dispatch_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_REAL64: {
                double tmp;
                dispatch_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void dispatch_sum( const NodeColumns& fs, const Field& field, std::vector<T>& result, idx_t& N ) {
    auto arr = make_leveled_view<const T>( field );
    const mesh::IsGhostNode is_ghost( fs.nodes() );
    const idx_t npts = std::min( arr.shape( 0 ), fs.nb_nodes() );
    const idx_t nlev = arr.shape( 1 );
    const idx_t nvar = arr.shape( 2 );
    std::vector<T> local_sum( nvar, 0 );
    result.resize( nvar );

    atlas_omp_parallel {
        std::vector<T> local_sum_private( nvar, 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            if ( !is_ghost( n ) ) {
                for ( idx_t l = 0; l < nlev; ++l ) {
                    for ( idx_t j = 0; j < nvar; ++j ) {
                        local_sum_private[j] += arr( n, l, j );
                    }
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t j = 0; j < nvar; ++j ) {
                local_sum[j] += local_sum_private[j];
            }
        }
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( local_sum, result, eckit::mpi::sum() ); }

    N = fs.nb_nodes_global() * nlev;
}

template <typename T>
void sum( const NodeColumns& fs, const Field& field, std::vector<T>& result, idx_t& N ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_sum( fs, field, result, N );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void dispatch_sum_per_level( const NodeColumns& fs, const Field& field, Field& sum, idx_t& N ) {
    mesh::IsGhostNode is_ghost( fs.nodes() );

    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    sum.resize( shape );

    auto arr = make_leveled_view<const T>( field );

    const idx_t npts = std::min( arr.shape( 0 ), fs.nb_nodes() );
    const idx_t nlev = arr.shape( 1 );
    const idx_t nvar = arr.shape( 2 );

    auto sum_per_level = make_per_level_view<T>( sum );

    for ( idx_t l = 0; l < sum_per_level.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < sum_per_level.shape( 1 ); ++j ) {
            sum_per_level( l, j ) = 0;
        }
    }

    atlas_omp_parallel {
        array::ArrayT<T> sum_per_level_private( sum_per_level.shape( 0 ), sum_per_level.shape( 1 ) );
        array::ArrayView<T, 2> sum_per_level_private_view = array::make_view<T, 2>( sum_per_level_private );

        for ( idx_t l = 0; l < sum_per_level_private_view.shape( 0 ); ++l ) {
            for ( idx_t j = 0; j < sum_per_level_private_view.shape( 1 ); ++j ) {
                sum_per_level_private_view( l, j ) = 0;
            }
        }

        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            if ( !is_ghost( n ) ) {
                for ( idx_t l = 0; l < nlev; ++l ) {
                    for ( idx_t j = 0; j < nvar; ++j ) {
                        sum_per_level_private_view( l, j ) += arr( n, l, j );
                    }
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t l = 0; l < sum_per_level_private.shape( 0 ); ++l ) {
                for ( idx_t j = 0; j < sum_per_level_private.shape( 1 ); ++j ) {
                    sum_per_level( l, j ) += sum_per_level_private_view( l, j );
                }
            }
        }
    }
    ATLAS_TRACE_MPI( ALLREDUCE ) {
        mpi::comm().allReduceInPlace( sum_per_level.data(), sum.size(), eckit::mpi::sum() );
    }
    N = fs.nb_nodes_global();
}

void sum_per_level( const NodeColumns& fs, const Field& field, Field& sum, idx_t& N ) {
    if ( field.datatype() != sum.datatype() ) {
        throw_Exception( "Field and sum are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_sum_per_level<int>( fs, field, sum, N );
        case array::DataType::KIND_INT64:
            return dispatch_sum_per_level<long>( fs, field, sum, N );
        case array::DataType::KIND_REAL32:
            return dispatch_sum_per_level<float>( fs, field, sum, N );
        case array::DataType::KIND_REAL64:
            return dispatch_sum_per_level<double>( fs, field, sum, N );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename DATATYPE>
void dispatch_order_independent_sum_2d( const NodeColumns& fs, const Field& field, DATATYPE& result, idx_t& N ) {
    idx_t root   = 0;
    Field global = fs.createField( field, option::global() );
    fs.gather( field, global );
    result   = 0;
    auto glb = array::make_view<DATATYPE, 1>( global );
    for ( idx_t jnode = 0; jnode < glb.size(); ++jnode ) {
        result += glb( jnode );
    }
    ATLAS_TRACE_MPI( BROADCAST ) { mpi::comm().broadcast( &result, 1, root ); }
    N = fs.nb_nodes_global();
}

template <typename T>
void dispatch_order_independent_sum( const NodeColumns& fs, const Field& field, T& result, idx_t& N ) {
    if ( field.levels() ) {
        auto arr = make_leveled_scalar_view<const T>( field );

        Field surface_field = fs.createField<T>( option::name( "surface" ) | option::levels( false ) );
        auto surface        = array::make_view<T, 1>( surface_field );

        const idx_t N0 = std::min( surface.shape( 0 ), arr.shape( 0 ) );
        const idx_t N1 = arr.shape( 1 );
        for ( idx_t n = 0; n < N0; ++n ) {
            surface( n ) = 0;
            for ( idx_t l = 0; l < N1; ++l ) {
                surface( n ) += arr( n, l );
            }
        }
        dispatch_order_independent_sum_2d( fs, surface_field, result, N );
        N *= arr.shape( 1 );
    }
    else {
        dispatch_order_independent_sum_2d( fs, field, result, N );
    }
}

template <typename T>
void order_independent_sum( const NodeColumns& fs, const Field& field, T& result, idx_t& N ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_order_independent_sum( fs, field, result, N );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                int tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_INT64: {
                long tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_REAL32: {
                float tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            case array::DataType::KIND_REAL64: {
                double tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result = tmp;
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename DATATYPE>
void dispatch_order_independent_sum_2d( const NodeColumns& fs, const Field& field, std::vector<DATATYPE>& result,
                                        idx_t& N ) {
    idx_t nvar = field.variables();
    result.resize( nvar );
    for ( idx_t j = 0; j < nvar; ++j ) {
        result[j] = 0.;
    }
    Field global = fs.createField( field, option::name( "global" ) | option::global() );
    fs.gather( field, global );
    if ( mpi::rank() == 0 ) {
        const auto glb = make_surface_view<DATATYPE>( global );
        for ( idx_t n = 0; n < fs.nb_nodes_global(); ++n ) {
            for ( idx_t j = 0; j < nvar; ++j ) {
                result[j] += glb( n, j );
            }
        }
    }
    idx_t root = global.metadata().get<idx_t>( "owner" );
    ATLAS_TRACE_MPI( BROADCAST ) { mpi::comm().broadcast( result, root ); }
    N = fs.nb_nodes_global();
}

template <typename T>
void dispatch_order_independent_sum( const NodeColumns& fs, const Field& field, std::vector<T>& result, idx_t& N ) {
    if ( field.levels() ) {
        const auto arr   = make_leveled_view<const T>( field );
        const idx_t npts = std::min( arr.shape( 0 ), fs.nb_nodes() );
        const idx_t nlev = arr.shape( 1 );
        const idx_t nvar = arr.shape( 2 );

        Field surface_field =
            fs.createField<T>( option::name( "surface" ) | option::variables( nvar ) | option::levels( false ) );
        auto surface = make_surface_view<T>( surface_field );

        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t j = 0; j < nvar; ++j ) {
                surface( n, j ) = 0;
            }
        }

        for ( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < nlev; ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    surface( n, j ) += arr( n, l, j );
                }
            }
        }

        dispatch_order_independent_sum_2d( fs, surface_field, result, N );
        N *= arr.shape( 1 );
    }
    else {
        dispatch_order_independent_sum_2d( fs, field, result, N );
    }
}

template <typename T>
void order_independent_sum( const NodeColumns& fs, const Field& field, std::vector<T>& result, idx_t& N ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_order_independent_sum( fs, field, result, N );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_order_independent_sum( fs, field, tmp, N );
                result.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void dispatch_order_independent_sum_per_level( const NodeColumns& fs, const Field& field, Field& sumfield, idx_t& N ) {
    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    sumfield.resize( shape );

    auto sum = make_per_level_view<T>( sumfield );
    for ( idx_t l = 0; l < sum.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < sum.shape( 1 ); ++j ) {
            sum( l, j ) = 0.;
        }
    }

    idx_t root   = 0;
    Field global = fs.createField( field, option::name( "global" ) | option::global() );

    fs.gather( field, global );
    if ( mpi::rank() == 0 ) {
        const array::LocalView<T, 3> glb = make_leveled_view<T>( global );

        for ( idx_t n = 0; n < glb.shape( 0 ); ++n ) {
            for ( idx_t l = 0; l < glb.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < glb.shape( 2 ); ++j ) {
                    sum( l, j ) += glb( n, l, j );
                }
            }
        }
    }
    ATLAS_TRACE_MPI( BROADCAST ) {
        std::vector<T> sum_array( sumfield.size() );
        if ( mpi::rank() == root ) {
            idx_t c( 0 );
            for ( idx_t l = 0; l < sum.shape( 0 ); ++l ) {
                for ( idx_t j = 0; j < sum.shape( 1 ); ++j ) {
                    sum_array[c++] = sum( l, j );
                }
            }
        }
        mpi::comm().broadcast( sum_array, root );
        if ( mpi::rank() != root ) {
            idx_t c( 0 );
            for ( idx_t l = 0; l < sum.shape( 0 ); ++l ) {
                for ( idx_t j = 0; j < sum.shape( 1 ); ++j ) {
                    sum( l, j ) = sum_array[c++];
                }
            }
        }
    }
    N = fs.nb_nodes_global();
}

void order_independent_sum_per_level( const NodeColumns& fs, const Field& field, Field& sum, idx_t& N ) {
    if ( field.datatype() != sum.datatype() ) {
        throw_Exception( "Field and sum are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_order_independent_sum_per_level<int>( fs, field, sum, N );
        case array::DataType::KIND_INT64:
            return dispatch_order_independent_sum_per_level<long>( fs, field, sum, N );
        case array::DataType::KIND_REAL32:
            return dispatch_order_independent_sum_per_level<float>( fs, field, sum, N );
        case array::DataType::KIND_REAL64:
            return dispatch_order_independent_sum_per_level<double>( fs, field, sum, N );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void dispatch_minimum( const NodeColumns& fs, const Field& field, std::vector<T>& min ) {
    auto arr         = make_leveled_view<const T>( field );
    const idx_t nvar = arr.shape( 2 );
    min.resize( nvar );
    std::vector<T> local_minimum( nvar, std::numeric_limits<T>::max() );
    atlas_omp_parallel {
        std::vector<T> local_minimum_private( nvar, std::numeric_limits<T>::max() );
        const idx_t npts = std::min( arr.shape( 0 ), fs.size() );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                    local_minimum_private[j] = std::min( arr( n, l, j ), local_minimum_private[j] );
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                local_minimum[j] = std::min( local_minimum_private[j], local_minimum[j] );
            }
        }
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( local_minimum, min, eckit::mpi::min() ); }
}

template <typename T>
void minimum( const NodeColumns& fs, const Field& field, std::vector<T>& min ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_minimum( fs, field, min );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_minimum( fs, field, tmp );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_minimum( fs, field, tmp );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_minimum( fs, field, tmp );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_minimum( fs, field, tmp );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void dispatch_maximum( const NodeColumns& fs, const Field& field, std::vector<T>& max ) {
    auto arr         = make_leveled_view<const T>( field );
    const idx_t nvar = arr.shape( 2 );
    max.resize( nvar );
    std::vector<T> local_maximum( nvar, -std::numeric_limits<T>::max() );
    atlas_omp_parallel {
        std::vector<T> local_maximum_private( nvar, -std::numeric_limits<T>::max() );
        const idx_t npts = std::min( arr.shape( 0 ), fs.size() );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    local_maximum_private[j] = std::max( arr( n, l, j ), local_maximum_private[j] );
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t j = 0; j < nvar; ++j ) {
                local_maximum[j] = std::max( local_maximum_private[j], local_maximum[j] );
            }
        }
    }
    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( local_maximum, max, eckit::mpi::max() ); }
}

template <typename T>
void maximum( const NodeColumns& fs, const Field& field, std::vector<T>& max ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_maximum( fs, field, max );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_maximum( fs, field, tmp );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_maximum( fs, field, tmp );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_maximum( fs, field, tmp );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_maximum( fs, field, tmp );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void minimum( const NodeColumns& fs, const Field& field, T& min ) {
    std::vector<T> v;
    minimum( fs, field, v );
    min = v[0];
}

template <typename T>
void maximum( const NodeColumns& fs, const Field& field, T& max ) {
    std::vector<T> v;
    maximum( fs, field, v );
    max = v[0];
}

template <typename T>
void dispatch_minimum_per_level( const NodeColumns& fs, const Field& field, Field& min_field ) {
    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    min_field.resize( shape );
    auto min = make_per_level_view<T>( min_field );

    for ( idx_t l = 0; l < min.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < min.shape( 1 ); ++j ) {
            min( l, j ) = std::numeric_limits<T>::max();
        }
    }

    auto arr = make_leveled_view<const T>( field );
    atlas_omp_parallel {
        array::ArrayT<T> min_private( min.shape( 0 ), min.shape( 1 ) );
        array::ArrayView<T, 2> min_private_view = array::make_view<T, 2>( min_private );
        for ( idx_t l = 0; l < min.shape( 0 ); ++l ) {
            for ( idx_t j = 0; j < min.shape( 1 ); ++j ) {
                min_private_view( l, j ) = std::numeric_limits<T>::max();
            }
        }

        const idx_t npts = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                    min_private_view( l, j ) = std::min( arr( n, l, j ), min_private_view( l, j ) );
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                    min( l, j ) = std::min( min_private_view( l, j ), min( l, j ) );
                }
            }
        }
    }
    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduceInPlace( min.data(), min_field.size(), eckit::mpi::min() ); }
}

void minimum_per_level( const NodeColumns& fs, const Field& field, Field& min ) {
    if ( field.datatype() != min.datatype() ) {
        throw_Exception( "Field and min are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_minimum_per_level<int>( fs, field, min );
        case array::DataType::KIND_INT64:
            return dispatch_minimum_per_level<long>( fs, field, min );
        case array::DataType::KIND_REAL32:
            return dispatch_minimum_per_level<float>( fs, field, min );
        case array::DataType::KIND_REAL64:
            return dispatch_minimum_per_level<double>( fs, field, min );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void dispatch_maximum_per_level( const NodeColumns&, const Field& field, Field& max_field ) {
    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    max_field.resize( shape );
    auto max = make_per_level_view<T>( max_field );

    for ( idx_t l = 0; l < max.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < max.shape( 1 ); ++j ) {
            max( l, j ) = -std::numeric_limits<T>::max();
        }
    }

    auto arr = make_leveled_view<const T>( field );
    atlas_omp_parallel {
        array::ArrayT<T> max_private( max.shape( 0 ), max.shape( 1 ) );
        array::ArrayView<T, 2> max_private_view = array::make_view<T, 2>( max_private );

        for ( idx_t l = 0; l < max_private_view.shape( 0 ); ++l ) {
            for ( idx_t j = 0; j < max_private_view.shape( 1 ); ++j ) {
                max_private_view( l, j ) = -std::numeric_limits<T>::max();
            }
        }

        const idx_t npts = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                    max_private_view( l, j ) = std::max( arr( n, l, j ), max_private_view( l, j ) );
                }
            }
        }
        atlas_omp_critical {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < arr.shape( 2 ); ++j ) {
                    max( l, j ) = std::max( max_private_view( l, j ), max( l, j ) );
                }
            }
        }
    }
    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduceInPlace( max.data(), max_field.size(), eckit::mpi::max() ); }
}

void maximum_per_level( const NodeColumns& fs, const Field& field, Field& max ) {
    if ( field.datatype() != max.datatype() ) {
        throw_Exception( "Field and max are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_maximum_per_level<int>( fs, field, max );
        case array::DataType::KIND_INT64:
            return dispatch_maximum_per_level<long>( fs, field, max );
        case array::DataType::KIND_REAL32:
            return dispatch_maximum_per_level<float>( fs, field, max );
        case array::DataType::KIND_REAL64:
            return dispatch_maximum_per_level<double>( fs, field, max );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void dispatch_minimum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& min,
                                    std::vector<gidx_t>& glb_idx, std::vector<idx_t>& level ) {
    auto arr   = make_leveled_view<const T>( field );
    idx_t nvar = arr.shape( 2 );
    min.resize( nvar );
    glb_idx.resize( nvar );
    level.resize( nvar );
    std::vector<T> local_minimum( nvar, std::numeric_limits<T>::max() );
    std::vector<idx_t> loc_node( nvar );
    std::vector<idx_t> loc_level( nvar );
    atlas_omp_parallel {
        std::vector<T> local_minimum_private( nvar, std::numeric_limits<T>::max() );
        std::vector<idx_t> loc_node_private( nvar );
        std::vector<idx_t> loc_level_private( nvar );
        const idx_t npts = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( arr( n, l, j ) < local_minimum_private[j] ) {
                        local_minimum_private[j] = arr( n, l, j );
                        loc_node_private[j]      = n;
                        loc_level_private[j]     = l;
                    }
                }
            }
        }
        atlas_omp_critical_ordered {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( local_minimum_private[j] < local_minimum[j] ) {
                        local_minimum[j] = local_minimum_private[j];
                        loc_node[j]      = loc_node_private[j];
                        loc_level[j]     = loc_level_private[j];
                    }
                }
            }
        }
    }
    std::vector<std::pair<T, int>> min_and_gidx_loc( nvar );
    std::vector<std::pair<T, int>> min_and_level_loc( nvar );
    std::vector<std::pair<T, int>> min_and_gidx_glb( nvar );
    std::vector<std::pair<T, int>> min_and_level_glb( nvar );
    const array::ArrayView<gidx_t, 1> global_index = array::make_view<gidx_t, 1>( fs.nodes().global_index() );
    for ( idx_t j = 0; j < nvar; ++j ) {
        gidx_t glb_idx = global_index( loc_node[j] );
        ATLAS_ASSERT( glb_idx < std::numeric_limits<int>::max() );  // pairs with 64bit
                                                                    // integer for second not
                                                                    // implemented
        min_and_gidx_loc[j]  = std::make_pair( local_minimum[j], glb_idx );
        min_and_level_loc[j] = std::make_pair( local_minimum[j], loc_level[j] );
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) {
        mpi::comm().allReduce( min_and_gidx_loc, min_and_gidx_glb, eckit::mpi::minloc() );
        mpi::comm().allReduce( min_and_level_loc, min_and_level_glb, eckit::mpi::minloc() );
    }

    for ( idx_t j = 0; j < nvar; ++j ) {
        min[j]     = min_and_gidx_glb[j].first;
        glb_idx[j] = min_and_gidx_glb[j].second;
        level[j]   = min_and_level_glb[j].second;
    }
}

template <typename T>
void minimum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& min, std::vector<gidx_t>& glb_idx,
                           std::vector<idx_t>& level ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_minimum_and_location( fs, field, min, glb_idx, level );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_minimum_and_location( fs, field, tmp, glb_idx, level );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_minimum_and_location( fs, field, tmp, glb_idx, level );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_minimum_and_location( fs, field, tmp, glb_idx, level );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_minimum_and_location( fs, field, tmp, glb_idx, level );
                min.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void dispatch_maximum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& max,
                                    std::vector<gidx_t>& glb_idx, std::vector<idx_t>& level ) {
    auto arr   = make_leveled_view<const T>( field );
    idx_t nvar = arr.shape( 2 );
    max.resize( nvar );
    glb_idx.resize( nvar );
    level.resize( nvar );
    std::vector<T> local_maximum( nvar, -std::numeric_limits<T>::max() );
    std::vector<idx_t> loc_node( nvar );
    std::vector<idx_t> loc_level( nvar );
    atlas_omp_parallel {
        std::vector<T> local_maximum_private( nvar, -std::numeric_limits<T>::max() );
        std::vector<idx_t> loc_node_private( nvar );
        std::vector<idx_t> loc_level_private( nvar );
        const idx_t npts = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( arr( n, l, j ) > local_maximum_private[j] ) {
                        local_maximum_private[j] = arr( n, l, j );
                        loc_node_private[j]      = n;
                        loc_level_private[j]     = l;
                    }
                }
            }
        }
        atlas_omp_critical_ordered {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( local_maximum_private[j] > local_maximum[j] ) {
                        local_maximum[j] = local_maximum_private[j];
                        loc_node[j]      = loc_node_private[j];
                        loc_level[j]     = loc_level_private[j];
                    }
                }
            }
        }
    }
    std::vector<std::pair<T, int>> max_and_gidx_loc( nvar );
    std::vector<std::pair<T, int>> max_and_level_loc( nvar );
    std::vector<std::pair<T, int>> max_and_gidx_glb( nvar );
    std::vector<std::pair<T, int>> max_and_level_glb( nvar );
    const array::ArrayView<gidx_t, 1> global_index = array::make_view<gidx_t, 1>( fs.nodes().global_index() );
    for ( idx_t j = 0; j < nvar; ++j ) {
        gidx_t glb_idx = global_index( loc_node[j] );
        ATLAS_ASSERT( glb_idx < std::numeric_limits<int>::max() );  // pairs with 64bit
                                                                    // integer for second not
                                                                    // implemented
        max_and_gidx_loc[j]  = std::make_pair( local_maximum[j], glb_idx );
        max_and_level_loc[j] = std::make_pair( local_maximum[j], loc_level[j] );
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) {
        mpi::comm().allReduce( max_and_gidx_loc, max_and_gidx_glb, eckit::mpi::maxloc() );
        mpi::comm().allReduce( max_and_level_loc, max_and_level_glb, eckit::mpi::maxloc() );
    }

    for ( idx_t j = 0; j < nvar; ++j ) {
        max[j]     = max_and_gidx_glb[j].first;
        glb_idx[j] = max_and_gidx_glb[j].second;
        level[j]   = max_and_level_glb[j].second;
    }
}

template <typename T>
void maximum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& max, std::vector<gidx_t>& glb_idx,
                           std::vector<idx_t>& level ) {
    if ( field.datatype() == array::DataType::kind<T>() ) {
        return dispatch_maximum_and_location( fs, field, max, glb_idx, level );
    }
    else {
        switch ( field.datatype().kind() ) {
            case array::DataType::KIND_INT32: {
                std::vector<int> tmp;
                dispatch_maximum_and_location( fs, field, tmp, glb_idx, level );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_INT64: {
                std::vector<long> tmp;
                dispatch_maximum_and_location( fs, field, tmp, glb_idx, level );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL32: {
                std::vector<float> tmp;
                dispatch_maximum_and_location( fs, field, tmp, glb_idx, level );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            case array::DataType::KIND_REAL64: {
                std::vector<double> tmp;
                dispatch_maximum_and_location( fs, field, tmp, glb_idx, level );
                max.assign( tmp.begin(), tmp.end() );
                return;
            }
            default:
                throw_Exception( "datatype not supported", Here() );
        }
    }
}

template <typename T>
void minimum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& min,
                           std::vector<gidx_t>& glb_idx ) {
    std::vector<idx_t> level;
    minimum_and_location( fs, field, min, glb_idx, level );
}

template <typename T>
void maximum_and_location( const NodeColumns& fs, const Field& field, std::vector<T>& max,
                           std::vector<gidx_t>& glb_idx ) {
    std::vector<idx_t> level;
    maximum_and_location( fs, field, max, glb_idx, level );
}

template <typename T>
void minimum_and_location( const NodeColumns& fs, const Field& field, T& min, gidx_t& glb_idx, idx_t& level ) {
    std::vector<T> minv;
    std::vector<gidx_t> gidxv;
    std::vector<idx_t> levelv;
    minimum_and_location( fs, field, minv, gidxv, levelv );
    min     = minv[0];
    glb_idx = gidxv[0];
    level   = levelv[0];
}

template <typename T>
void maximum_and_location( const NodeColumns& fs, const Field& field, T& max, gidx_t& glb_idx, idx_t& level ) {
    std::vector<T> maxv;
    std::vector<gidx_t> gidxv;
    std::vector<idx_t> levelv;
    maximum_and_location( fs, field, maxv, gidxv, levelv );
    max     = maxv[0];
    glb_idx = gidxv[0];
    level   = levelv[0];
}

template <typename T>
void minimum_and_location( const NodeColumns& fs, const Field& field, T& min, gidx_t& glb_idx ) {
    idx_t level;
    minimum_and_location( fs, field, min, glb_idx, level );
}

template <typename T>
void maximum_and_location( const NodeColumns& fs, const Field& field, T& max, gidx_t& glb_idx ) {
    idx_t level;
    maximum_and_location( fs, field, max, glb_idx, level );
}

template <typename T>
void dispatch_minimum_and_location_per_level( const NodeColumns& fs, const Field& field, Field& min_field,
                                              Field& glb_idx_field ) {
    auto arr = make_leveled_view<const T>( field );
    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    min_field.resize( shape );
    glb_idx_field.resize( shape );
    const idx_t nvar = arr.shape( 2 );
    auto min         = make_per_level_view<T>( min_field );
    auto glb_idx     = make_per_level_view<gidx_t>( glb_idx_field );

    for ( idx_t l = 0; l < min.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < min.shape( 1 ); ++j ) {
            min( l, j ) = std::numeric_limits<T>::max();
        }
    }

    atlas_omp_parallel {
        array::ArrayT<T> min_private( min.shape( 0 ), min.shape( 1 ) );
        array::ArrayView<T, 2> min_private_view = array::make_view<T, 2>( min_private );

        for ( idx_t l = 0; l < min_private_view.shape( 0 ); ++l ) {
            for ( idx_t j = 0; j < min_private_view.shape( 1 ); ++j ) {
                min_private_view( l, j ) = std::numeric_limits<T>::max();
            }
        }

        array::ArrayT<gidx_t> glb_idx_private( glb_idx.shape( 0 ), glb_idx.shape( 1 ) );
        array::ArrayView<gidx_t, 2> glb_idx_private_view = array::make_view<gidx_t, 2>( glb_idx_private );
        const idx_t npts                                 = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( arr( n, l, j ) < min( l, j ) ) {
                        min_private_view( l, j )     = arr( n, l, j );
                        glb_idx_private_view( l, j ) = n;
                    }
                }
            }
        }
        atlas_omp_critical_ordered {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( min_private_view( l, j ) < min( l, j ) ) {
                        min( l, j )     = min_private_view( l, j );
                        glb_idx( l, j ) = glb_idx_private_view( l, j );
                    }
                }
            }
        }
    }
    const idx_t nlev = arr.shape( 1 );
    std::vector<std::pair<T, int>> min_and_gidx_loc( nlev * nvar );
    std::vector<std::pair<T, int>> min_and_gidx_glb( nlev * nvar );
    const array::ArrayView<gidx_t, 1> global_index = array::make_view<gidx_t, 1>( fs.nodes().global_index() );
    atlas_omp_parallel_for( idx_t l = 0; l < nlev; ++l ) {
        for ( idx_t j = 0; j < nvar; ++j ) {
            gidx_t gidx = global_index( glb_idx( l, j ) );
            ATLAS_ASSERT( gidx < std::numeric_limits<int>::max() );  // pairs with 64bit
                                                                     // integer for second not
                                                                     // implemented
            min_and_gidx_loc[j + nvar * l] = std::make_pair( min( l, j ), gidx );
        }
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( min_and_gidx_loc, min_and_gidx_glb, eckit::mpi::minloc() ); }

    atlas_omp_parallel_for( idx_t l = 0; l < nlev; ++l ) {
        for ( idx_t j = 0; j < nvar; ++j ) {
            min( l, j )     = min_and_gidx_glb[j + l * nvar].first;
            glb_idx( l, j ) = min_and_gidx_glb[j + l * nvar].second;
        }
    }
}

void minimum_and_location_per_level( const NodeColumns& fs, const Field& field, Field& min, Field& glb_idx ) {
    if ( field.datatype() != min.datatype() ) {
        throw_Exception( "Field and min are not of same datatype.", Here() );
    }
    if ( glb_idx.datatype() != array::DataType::kind<gidx_t>() ) {
        throw_Exception( "glb_idx Field is not of correct datatype", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_minimum_and_location_per_level<int>( fs, field, min, glb_idx );
        case array::DataType::KIND_INT64:
            return dispatch_minimum_and_location_per_level<long>( fs, field, min, glb_idx );
        case array::DataType::KIND_REAL32:
            return dispatch_minimum_and_location_per_level<float>( fs, field, min, glb_idx );
        case array::DataType::KIND_REAL64:
            return dispatch_minimum_and_location_per_level<double>( fs, field, min, glb_idx );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void dispatch_maximum_and_location_per_level( const NodeColumns& fs, const Field& field, Field& max_field,
                                              Field& glb_idx_field ) {
    auto arr = make_leveled_view<const T>( field );
    array::ArrayShape shape;
    shape.reserve( field.rank() - 1 );
    for ( idx_t j = 1; j < field.rank(); ++j ) {
        shape.push_back( field.shape( j ) );
    }
    max_field.resize( shape );
    glb_idx_field.resize( shape );
    const idx_t nvar = arr.shape( 2 );
    auto max         = make_per_level_view<T>( max_field );
    auto glb_idx     = make_per_level_view<gidx_t>( glb_idx_field );

    for ( idx_t l = 0; l < max.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < max.shape( 1 ); ++j ) {
            max( l, j ) = -std::numeric_limits<T>::max();
        }
    }

    atlas_omp_parallel {
        array::ArrayT<T> max_private( max.shape( 0 ), max.shape( 1 ) );
        array::ArrayView<T, 2> max_private_view = array::make_view<T, 2>( max_private );

        for ( idx_t l = 0; l < max_private_view.shape( 0 ); ++l ) {
            for ( idx_t j = 0; j < max_private_view.shape( 1 ); ++j ) {
                max_private_view( l, j ) = -std::numeric_limits<T>::max();
            }
        }

        array::ArrayT<gidx_t> glb_idx_private( glb_idx.shape( 0 ), glb_idx.shape( 1 ) );
        array::ArrayView<gidx_t, 2> glb_idx_private_view = array::make_view<gidx_t, 2>( glb_idx_private );
        const idx_t npts                                 = arr.shape( 0 );
        atlas_omp_for( idx_t n = 0; n < npts; ++n ) {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( arr( n, l, j ) > max( l, j ) ) {
                        max_private_view( l, j )     = arr( n, l, j );
                        glb_idx_private_view( l, j ) = n;
                    }
                }
            }
        }
        atlas_omp_critical_ordered {
            for ( idx_t l = 0; l < arr.shape( 1 ); ++l ) {
                for ( idx_t j = 0; j < nvar; ++j ) {
                    if ( max_private_view( l, j ) > max( l, j ) ) {
                        max( l, j )     = max_private_view( l, j );
                        glb_idx( l, j ) = glb_idx_private_view( l, j );
                    }
                }
            }
        }
    }

    const idx_t nlev = arr.shape( 1 );
    std::vector<std::pair<T, int>> max_and_gidx_loc( nlev * nvar );
    std::vector<std::pair<T, int>> max_and_gidx_glb( nlev * nvar );
    const array::ArrayView<gidx_t, 1> global_index = array::make_view<gidx_t, 1>( fs.nodes().global_index() );
    atlas_omp_parallel_for( idx_t l = 0; l < nlev; ++l ) {
        for ( idx_t j = 0; j < nvar; ++j ) {
            gidx_t gidx = global_index( glb_idx( l, j ) );
            ATLAS_ASSERT( gidx < std::numeric_limits<int>::max() );  // pairs with 64bit
                                                                     // integer for second not
                                                                     // implemented
            max_and_gidx_loc[j + nvar * l] = std::make_pair( max( l, j ), gidx );
        }
    }

    ATLAS_TRACE_MPI( ALLREDUCE ) { mpi::comm().allReduce( max_and_gidx_loc, max_and_gidx_glb, eckit::mpi::maxloc() ); }

    atlas_omp_parallel_for( idx_t l = 0; l < nlev; ++l ) {
        for ( idx_t j = 0; j < nvar; ++j ) {
            max( l, j )     = max_and_gidx_glb[j + l * nvar].first;
            glb_idx( l, j ) = max_and_gidx_glb[j + l * nvar].second;
        }
    }
}

void maximum_and_location_per_level( const NodeColumns& fs, const Field& field, Field& max, Field& glb_idx ) {
    if ( field.datatype() != max.datatype() ) {
        throw_Exception( "Field and max are not of same datatype.", Here() );
    }
    if ( glb_idx.datatype() != array::DataType::kind<gidx_t>() ) {
        throw_Exception( "glb_idx Field is not of correct datatype", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_maximum_and_location_per_level<int>( fs, field, max, glb_idx );
        case array::DataType::KIND_INT64:
            return dispatch_maximum_and_location_per_level<long>( fs, field, max, glb_idx );
        case array::DataType::KIND_REAL32:
            return dispatch_maximum_and_location_per_level<float>( fs, field, max, glb_idx );
        case array::DataType::KIND_REAL64:
            return dispatch_maximum_and_location_per_level<double>( fs, field, max, glb_idx );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void mean( const NodeColumns& fs, const Field& field, T& result, idx_t& N ) {
    sum( fs, field, result, N );
    result /= static_cast<double>( N );
}

template <typename T>
void mean( const NodeColumns& fs, const Field& field, std::vector<T>& result, idx_t& N ) {
    sum( fs, field, result, N );
    for ( size_t j = 0; j < result.size(); ++j ) {
        result[j] /= static_cast<double>( N );
    }
}

template <typename T>
void dispatch_mean_per_level( const NodeColumns& fs, const Field& field, Field& mean, idx_t& N ) {
    dispatch_sum_per_level<T>( fs, field, mean, N );
    auto view = make_per_level_view<T>( mean );
    for ( idx_t l = 0; l < view.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < view.shape( 1 ); ++j ) {
            view( l, j ) /= static_cast<double>( N );
        }
    }
}

void mean_per_level( const NodeColumns& fs, const Field& field, Field& mean, idx_t& N ) {
    if ( field.datatype() != mean.datatype() ) {
        throw_Exception( "Field and sum are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_mean_per_level<int>( fs, field, mean, N );
        case array::DataType::KIND_INT64:
            return dispatch_mean_per_level<long>( fs, field, mean, N );
        case array::DataType::KIND_REAL32:
            return dispatch_mean_per_level<float>( fs, field, mean, N );
        case array::DataType::KIND_REAL64:
            return dispatch_mean_per_level<double>( fs, field, mean, N );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

template <typename T>
void mean_and_standard_deviation( const NodeColumns& fs, const Field& field, T& mu, T& sigma, idx_t& N ) {
    mean( fs, field, mu, N );
    Field squared_diff_field = fs.createField( option::name( "sqr_diff" ) | option::datatype( field.datatype() ) |
                                               option::levels( field.levels() ) );

    auto squared_diff = make_leveled_scalar_view<T>( squared_diff_field );
    auto values       = make_leveled_scalar_view<const T>( field );

    const idx_t npts = std::min<idx_t>( values.shape( 0 ), fs.nb_nodes() );
    atlas_omp_parallel_for( idx_t n = 0; n < npts; ++n ) {
        for ( idx_t l = 0; l < values.shape( 1 ); ++l ) {
            squared_diff( n, l ) = sqr( values( n, l ) - mu );
        }
    }
    mean( fs, squared_diff_field, sigma, N );
    sigma = std::sqrt( sigma );
}

template <typename T>
void mean_and_standard_deviation( const NodeColumns& fs, const Field& field, std::vector<T>& mu, std::vector<T>& sigma,
                                  idx_t& N ) {
    mean( fs, field, mu, N );
    Field squared_diff_field = fs.createField<T>( option::name( "sqr_diff" ) | option::levels( field.levels() ) |
                                                  option::variables( field.variables() ) );
    auto squared_diff        = make_leveled_view<T>( squared_diff_field );
    auto values              = make_leveled_view<const T>( field );

    const idx_t npts = std::min<idx_t>( values.shape( 0 ), fs.nb_nodes() );
    atlas_omp_parallel_for( idx_t n = 0; n < npts; ++n ) {
        for ( idx_t l = 0; l < values.shape( 1 ); ++l ) {
            for ( idx_t j = 0; j < values.shape( 2 ); ++j ) {
                squared_diff( n, l, j ) = sqr( values( n, l, j ) - mu[j] );
            }
        }
    }
    mean( fs, squared_diff_field, sigma, N );
    for ( size_t j = 0; j < sigma.size(); ++j ) {
        sigma[j] = std::sqrt( sigma[j] );
    }
}

template <typename T>
void dispatch_mean_and_standard_deviation_per_level( const NodeColumns& fs, const Field& field, Field& mean,
                                                     Field& stddev, idx_t& N ) {
    dispatch_mean_per_level<T>( fs, field, mean, N );
    Field squared_diff_field = fs.createField<T>( option::name( "sqr_diff" ) | option::levels( field.levels() ) |
                                                  option::variables( field.variables() ) );
    auto squared_diff        = make_leveled_view<T>( squared_diff_field );
    auto values              = make_leveled_view<const T>( field );
    auto mu                  = make_per_level_view<T>( mean );

    const idx_t npts = std::min<idx_t>( values.shape( 0 ), fs.nb_nodes() );
    atlas_omp_parallel_for( idx_t n = 0; n < npts; ++n ) {
        for ( idx_t l = 0; l < values.shape( 1 ); ++l ) {
            for ( idx_t j = 0; j < values.shape( 2 ); ++j ) {
                squared_diff( n, l, j ) = sqr( values( n, l, j ) - mu( l, j ) );
            }
        }
    }
    dispatch_mean_per_level<T>( fs, squared_diff_field, stddev, N );
    auto sigma = make_per_level_view<T>( stddev );
    atlas_omp_for( idx_t l = 0; l < sigma.shape( 0 ); ++l ) {
        for ( idx_t j = 0; j < sigma.shape( 1 ); ++j ) {
            sigma( l, j ) = std::sqrt( sigma( l, j ) );
        }
    }
}

void mean_and_standard_deviation_per_level( const NodeColumns& fs, const Field& field, Field& mean, Field& stddev,
                                            idx_t& N ) {
    if ( field.datatype() != mean.datatype() ) {
        throw_Exception( "Field and mean are not of same datatype.", Here() );
    }
    if ( field.datatype() != stddev.datatype() ) {
        throw_Exception( "Field and stddev are not of same datatype.", Here() );
    }
    switch ( field.datatype().kind() ) {
        case array::DataType::KIND_INT32:
            return dispatch_mean_and_standard_deviation_per_level<int>( fs, field, mean, stddev, N );
        case array::DataType::KIND_INT64:
            return dispatch_mean_and_standard_deviation_per_level<long>( fs, field, mean, stddev, N );
        case array::DataType::KIND_REAL32:
            return dispatch_mean_and_standard_deviation_per_level<float>( fs, field, mean, stddev, N );
        case array::DataType::KIND_REAL64:
            return dispatch_mean_and_standard_deviation_per_level<double>( fs, field, mean, stddev, N );
        default:
            throw_Exception( "datatype not supported", Here() );
    }
}

}  // namespace detail

template <typename Value>
NodeColumns::FieldStatisticsT<Value>::FieldStatisticsT( const NodeColumns* f ) : functionspace( *f ) {}

template <typename Vector>
NodeColumns::FieldStatisticsVectorT<Vector>::FieldStatisticsVectorT( const NodeColumns* f ) : functionspace( *f ) {}

NodeColumns::FieldStatistics::FieldStatistics( const NodeColumns* f ) : functionspace( *f ) {}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::sum( const Field& field, Value& result, idx_t& N ) const {
    detail::sum( functionspace, field, result, N );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::sum( const Field& field, Vector& result, idx_t& N ) const {
    detail::sum( functionspace, field, result, N );
}

void NodeColumns::FieldStatistics::sumPerLevel( const Field& field, Field& result, idx_t& N ) const {
    detail::sum_per_level( functionspace, field, result, N );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::orderIndependentSum( const Field& field, Value& result, idx_t& N ) const {
    detail::order_independent_sum( functionspace, field, result, N );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::orderIndependentSum( const Field& field, Vector& result,
                                                                       idx_t& N ) const {
    detail::order_independent_sum( functionspace, field, result, N );
}

void NodeColumns::FieldStatistics::orderIndependentSumPerLevel( const Field& field, Field& result, idx_t& N ) const {
    detail::order_independent_sum_per_level( functionspace, field, result, N );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::minimum( const Field& field, Value& minimum ) const {
    detail::minimum( functionspace, field, minimum );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::maximum( const Field& field, Value& maximum ) const {
    detail::maximum( functionspace, field, maximum );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::minimum( const Field& field, Vector& minimum ) const {
    detail::minimum( functionspace, field, minimum );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::maximum( const Field& field, Vector& maximum ) const {
    detail::maximum( functionspace, field, maximum );
}

void NodeColumns::FieldStatistics::minimumPerLevel( const Field& field, Field& minimum ) const {
    detail::minimum_per_level( functionspace, field, minimum );
}

void NodeColumns::FieldStatistics::maximumPerLevel( const Field& field, Field& maximum ) const {
    detail::maximum_per_level( functionspace, field, maximum );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::minimumAndLocation( const Field& field, Value& minimum,
                                                               gidx_t& glb_idx ) const {
    detail::minimum_and_location( functionspace, field, minimum, glb_idx );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::maximumAndLocation( const Field& field, Value& maximum,
                                                               gidx_t& glb_idx ) const {
    detail::maximum_and_location( functionspace, field, maximum, glb_idx );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::minimumAndLocation( const Field& field, Value& minimum, gidx_t& glb_idx,
                                                               idx_t& level ) const {
    detail::minimum_and_location( functionspace, field, minimum, glb_idx, level );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::maximumAndLocation( const Field& field, Value& maximum, gidx_t& glb_idx,
                                                               idx_t& level ) const {
    detail::maximum_and_location( functionspace, field, maximum, glb_idx, level );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::minimumAndLocation( const Field& field, Vector& minimum,
                                                                      std::vector<gidx_t>& glb_idx ) const {
    detail::minimum_and_location( functionspace, field, minimum, glb_idx );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::maximumAndLocation( const Field& field, Vector& maximum,
                                                                      std::vector<gidx_t>& glb_idx ) const {
    detail::maximum_and_location( functionspace, field, maximum, glb_idx );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::minimumAndLocation( const Field& field, Vector& minimum,
                                                                      std::vector<gidx_t>& glb_idx,
                                                                      std::vector<idx_t>& level ) const {
    detail::minimum_and_location( functionspace, field, minimum, glb_idx, level );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::maximumAndLocation( const Field& field, Vector& maximum,
                                                                      std::vector<gidx_t>& glb_idx,
                                                                      std::vector<idx_t>& level ) const {
    detail::maximum_and_location( functionspace, field, maximum, glb_idx, level );
}

void NodeColumns::FieldStatistics::minimumAndLocationPerLevel( const Field& field, Field& column,
                                                               Field& glb_idx ) const {
    detail::minimum_and_location_per_level( functionspace, field, column, glb_idx );
}

void NodeColumns::FieldStatistics::maximumAndLocationPerLevel( const Field& field, Field& column,
                                                               Field& glb_idx ) const {
    detail::maximum_and_location_per_level( functionspace, field, column, glb_idx );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::mean( const Field& field, Value& mean, idx_t& N ) const {
    detail::mean( functionspace, field, mean, N );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::mean( const Field& field, Vector& mean, idx_t& N ) const {
    detail::mean( functionspace, field, mean, N );
}

void NodeColumns::FieldStatistics::meanPerLevel( const Field& field, Field& mean, idx_t& N ) const {
    detail::mean_per_level( functionspace, field, mean, N );
}

template <typename Value>
void NodeColumns::FieldStatisticsT<Value>::meanAndStandardDeviation( const Field& field, Value& mean, Value& stddev,
                                                                     idx_t& N ) const {
    detail::mean_and_standard_deviation( functionspace, field, mean, stddev, N );
}

template <typename Vector>
void NodeColumns::FieldStatisticsVectorT<Vector>::meanAndStandardDeviation( const Field& field, Vector& mean,
                                                                            Vector& stddev, idx_t& N ) const {
    detail::mean_and_standard_deviation( functionspace, field, mean, stddev, N );
}

void NodeColumns::FieldStatistics::meanAndStandardDeviationPerLevel( const Field& field, Field& mean, Field& stddev,
                                                                     idx_t& N ) const {
    detail::mean_and_standard_deviation_per_level( functionspace, field, mean, stddev, N );
}

template struct NodeColumns::FieldStatisticsT<int>;
template struct NodeColumns::FieldStatisticsT<long>;
template struct NodeColumns::FieldStatisticsT<float>;
template struct NodeColumns::FieldStatisticsT<double>;
// template struct NodeColumns::FieldStatisticsT<unsigned long>;
template struct NodeColumns::FieldStatisticsVectorT<std::vector<int>>;
template struct NodeColumns::FieldStatisticsVectorT<std::vector<long>>;
template struct NodeColumns::FieldStatisticsVectorT<std::vector<float>>;
template struct NodeColumns::FieldStatisticsVectorT<std::vector<double>>;
// template struct NodeColumns::FieldStatisticsVectorT< std::vector<unsigned
// long> >;

}  // namespace detail

}  // namespace functionspace
}  // namespace atlas
