#ifdef __MINGW32__
#define _hypot hypot
#include <cmath>
#endif

#if defined(_WINDOWS) && defined(_DEBUG)
// Not having a debug build of numpy shouldn't stop
// me being able to debug the other parts of the library
#define IISIGNATURE_NO_NUMPY
#endif

#include <Python.h>
#include <iostream>
#include <limits>
#include <memory>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
#ifndef IISIGNATURE_NO_NUMPY
#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
#include <numpy/arrayobject.h>
#endif

#include "calcSignature.hpp"
#include "logSigLength.hpp"
#include "logsig.hpp"
#include "rotationalInvariants.hpp"

using std::vector;

// for python 3.5+, we can load the new way
// https://www.python.org/dev/peps/pep-0489/

/*
template<typename T> int numpyTypenum;
template<> int numpyTypenum<float>  = NPY_FLOAT32;
template<> int numpyTypenum<double>  = NPY_FLOAT64;

template<int> class numpyTypeImpl;
template<> class numpyTypeImpl<NPY_FLOAT32>{public: typedef float Type;};
template<> class numpyTypeImpl<NPY_FLOAT64>{public: typedef double Type;};
template<int i> using numpyNumToType = typename numpyTypeImpl<i>::Type;
*/

#define ERR(X)                                                                 \
  {                                                                            \
    PyErr_SetString(PyExc_RuntimeError, X);                                    \
    return nullptr;                                                            \
  }
#define ERRb(X)                                                                \
  {                                                                            \
    PyErr_SetString(PyExc_RuntimeError, X);                                    \
    return false;                                                              \
  }
#define ERRr(X)                                                                \
  {                                                                            \
    PyErr_SetString(PyExc_RuntimeError, X);                                    \
    return;                                                                    \
  }

// INTERRUPTS - the following seems to be fine even on windows
#include <signal.h>
volatile bool g_signals_setup = false;
volatile bool g_signal_given = false;
void catcher(int) {
  g_signal_given = true;
  PyErr_SetInterrupt();
}

void setup_signals() {
  g_signal_given = false;
  if (g_signals_setup)
    return;
  signal(SIGINT, &catcher);
  g_signals_setup = true;
}
bool interrupt_wanted() { return g_signal_given; }
void interrupt() {
  if (g_signal_given)
    throw std::runtime_error("INTERRUPTION");
}

// The following 3 functions just do t, which returns true if it succeeded,
//  returning true if t succeeded.
// When in one of the python functions (defined in this file),
// calling a function which might call interrupt needs catching like this.
template <typename T> bool do_interruptible(T &&t) {
  bool ok = false;
  setup_signals();
  std::string exceptionMessage;
  try {
    ok = t();
  } catch (std::exception &e) {
    exceptionMessage = e.what();
  }
  if (PyErr_CheckSignals())
    return false;
  if (!exceptionMessage.empty())
    ERRb(exceptionMessage.c_str());
  return ok;
}
template <typename T> bool do_dummy(T &&t) { // useful for timing experiments,
  return t();
}
// Like do_interruptible but also releases the GIL
template <typename T> bool do_interruptible_releasing_lock(T &&t) {
  bool ok = false;
  setup_signals();
  std::string exceptionMessage;
  Py_BEGIN_ALLOW_THREADS try { ok = t(); } catch (std::exception &e) {
    exceptionMessage = e.what();
  }
  Py_END_ALLOW_THREADS if (PyErr_CheckSignals()) return false;
  if (!exceptionMessage.empty())
    ERRb(exceptionMessage.c_str());
  return ok;
}

// end of Interrupts stuff

// This class is constructed from a PyObject* and decreases
// its reference count on destruction.
// It is basically an "owner" of a reference.
class RefHolder {
  PyObject *m_p;

public:
  explicit RefHolder(PyObject *p) : m_p(p) {};
  RefHolder(const RefHolder &) = delete;
  RefHolder operator=(const RefHolder &) = delete;
  ~RefHolder() { Py_DECREF(m_p); }
};

// This is a version which can change what it owns (or whether it owns
// anything).
class ReleasableRefHolder {
  PyObject *m_p;

public:
  explicit ReleasableRefHolder(PyObject *p) : m_p(p) {};
  ReleasableRefHolder() : m_p(nullptr) {};
  void releaseAndSet(PyObject *p) { m_p = p; }
  void release() { m_p = nullptr; }
  ReleasableRefHolder(const ReleasableRefHolder &) = delete;
  ReleasableRefHolder operator=(const ReleasableRefHolder &) = delete;
  ~ReleasableRefHolder() { Py_XDECREF(m_p); }
};

#ifndef IISIGNATURE_NO_NUMPY
struct UseFloat {
  constexpr static int typenum = NPY_FLOAT32;
  using T = float;
};

struct UseDouble {
  constexpr static int typenum = NPY_FLOAT64;
  using T = double;
};

// Returns a new numpy array with the type typenum,
// with ndims dimensions. The first (ndims-1) dimensions are the same
// as those of arr, and the last dimension is newLastDim.
// arr must have at least one dimension.
PyObject *simpleNew_ownLastDim(int ndims, PyArrayObject *arr, size_t newLastDim,
                               int typenum) {
  npy_intp *shape = PyArray_DIMS(arr);
  npy_intp orig = shape[ndims - 1];
  shape[ndims - 1] = (npy_intp)newLastDim;
  PyObject *o = PyArray_SimpleNew(ndims, shape, typenum);
  shape[ndims - 1] = orig;
  return o;
}

// Returns a new numpy array with the type typenum,
// with ndims dimensions. The first (ndims-2) dimensions are the same
// as those of arr, and the last dimensions are newPenultDim and newLastDim.
// arr must have at least two dimensions.
PyObject *simpleNew_ownLast2Dims(int ndims, PyArrayObject *arr,
                                 size_t newPenultDim, size_t newLastDim,
                                 int typenum) {
  npy_intp *shape = PyArray_DIMS(arr);
  npy_intp orig1 = shape[ndims - 2];
  npy_intp orig2 = shape[ndims - 1];
  shape[ndims - 2] = (npy_intp)newPenultDim;
  shape[ndims - 1] = (npy_intp)newLastDim;
  PyObject *o = PyArray_SimpleNew(ndims, shape, typenum);
  shape[ndims - 2] = orig1;
  shape[ndims - 1] = orig2;
  return o;
}

// Check whether the last `nInner` dimensions of `a` are C-contiguous.
static bool isInnerContiguous(PyArrayObject *a, int nInner) {
  int nd = PyArray_NDIM(a);
  npy_intp *strides = PyArray_STRIDES(a);
  npy_intp *dims = PyArray_DIMS(a);
  npy_intp expected = (npy_intp)PyArray_ITEMSIZE(a);
  for (int i = nd - 1; i >= nd - nInner; --i) {
    if (strides[i] != expected)
      return false;
    expected *= dims[i];
  }
  return true;
}

// Given batch index `path` (0..nPaths-1), compute pointer into `a`
// skipping the last `nInner` dimensions. Uses odometer-style decomposition.
static char *batchPtr(PyArrayObject *a, int nInner, int path) {
  int nd = PyArray_NDIM(a);
  npy_intp *strides = PyArray_STRIDES(a);
  npy_intp *dims = PyArray_DIMS(a);
  char *ptr = (char *)PyArray_DATA(a);
  for (int i = nd - nInner - 1; i >= 0; --i) {
    npy_intp idx = path % dims[i];
    path /= dims[i];
    ptr += idx * strides[i];
  }
  return ptr;
}
#endif

#define STRINGIFY(x) #x
#define TOSTRING(x) STRINGIFY(x)
static PyObject *version(PyObject *self, PyObject *args) {
  // consider returning build time?
  return PyUnicode_FromString(TOSTRING(VERSION));
}

static PyObject *siglength(PyObject *self, PyObject *args) {
  int d = 0, m = 0;
  if (!PyArg_ParseTuple(args, "ii", &d, &m))
    return nullptr;
  if (m < 1)
    ERR("level must be positive");
  if (d < 1)
    ERR("dimension must be positive");
  long ans = calcSigTotalLength(d, m);
  // todo - cope with overrun here
  return PyLong_FromLong(ans);
}

static PyObject *logsiglength(PyObject *self, PyObject *args) {
  int d = 0, m = 0;
  if (!PyArg_ParseTuple(args, "ii", &d, &m))
    return nullptr;
  if (m < 1)
    ERR("level must be positive");
  if (d < 1)
    ERR("dimension must be positive");
  LogSigLength::Int ans = d == 1 ? 1
                          : m == 1
                              ? d
                              : LogSigLength::countNecklacesUptoLengthM(d, m);
  if (ans > std::numeric_limits<long>::max())
    ERR("overflow");
  // todo - cope with overrun here
  return PyLong_FromLong((long)ans);
}

#ifndef IISIGNATURE_NO_NUMPY
// returns true on success
// makes s2 be the signature of the path in data
// data is (lengthOfPath x d) with given strides
using CalcSignature::Signature;
template <typename InputT>
static bool calcSignature(Signature &s2, const char *data,
                          npy_intp point_stride, npy_intp dim_stride,
                          int lengthOfPath, int d, int level) {
  Signature s1;

  if (lengthOfPath == 1) {
    s2.sigOfNothing(d, level);
  }

  vector<double> displacement(d);
  for (int i = 1; i < lengthOfPath; ++i) {
    for (int j = 0; j < d; ++j)
      displacement[j] =
          *(InputT *)(data + i * point_stride + j * dim_stride) -
          *(InputT *)(data + (i - 1) * point_stride + j * dim_stride);
    s1.sigOfSegment(d, level, &displacement[0]);
    if (interrupt_wanted())
      return false;
    if (i == 1)
      s2.swap(s1);
    else {
      s2.concatenateWith(d, level, s1);
      if (interrupt_wanted())
        return false;
    }
  }
  return true;
}

// data is (lengthOfPath x d) with given strides
// out is ((lengthOfPath-1) x siglength)
// sets out[i] to signature of data[0:i,:]
template <typename InputT>
static bool calcCumulativeSignature(const char *data, npy_intp point_stride,
                                    npy_intp dim_stride, int lengthOfPath,
                                    int d, int level, size_t siglength,
                                    double *out) {
  Signature s1, s2;

  vector<double> displacement(d);
  for (int i = 1; i < lengthOfPath; ++i) {
    for (int j = 0; j < d; ++j)
      displacement[j] =
          *(InputT *)(data + i * point_stride + j * dim_stride) -
          *(InputT *)(data + (i - 1) * point_stride + j * dim_stride);
    s1.sigOfSegment(d, level, &displacement[0]);
    if (interrupt_wanted())
      return false;
    if (i == 1)
      s2.swap(s1);
    else {
      s2.concatenateWith(d, level, s1);
      if (interrupt_wanted())
        return false;
    }
    s2.writeOut(out + (i - 1) * siglength);
  }
  return true;
}

static PyObject *sig(PyObject *self, PyObject *args) {
  PyObject *a1;
  int level = 0;
  int format = 0;
  if (!PyArg_ParseTuple(args, "Oi|i", &a1, &level, &format))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  bool is_float32 =
      PyArray_Check(a1) && PyArray_TYPE((PyArrayObject *)a1) == NPY_FLOAT32;
  PyObject *aa = PyArray_FROM_OTF(a1, is_float32 ? NPY_FLOAT32 : NPY_FLOAT64,
                                  NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  bool use_float32 = (PyArray_TYPE(a) == NPY_FLOAT32);
  int ndims = PyArray_NDIM(a);
  if (ndims < 2)
    ERR("data must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, ndims - 2);
  const int d = (int)PyArray_DIM(a, ndims - 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d < 1)
    ERR("Path must have positive dimension");
  int nPaths = 1;
  for (int i = 0; i + 2 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    nPaths *= (int)x;
  }
  size_t eachOutputSize = (size_t)calcSigTotalLength(d, level);
  npy_intp *strides = PyArray_STRIDES(a);
  npy_intp point_stride = strides[ndims - 2];
  npy_intp dim_stride = strides[ndims - 1];
  PyObject *o = nullptr;
  using OutT = UseDouble;
  OutT::T *out_data = nullptr;
  if (format == 2) {
    o = simpleNew_ownLast2Dims(ndims, a, (size_t)(lengthOfPath - 1u),
                               eachOutputSize, OutT::typenum);
    if (!o)
      return nullptr;
    auto od = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
    auto doCalcCumul = [&](auto dummy) -> bool {
      using InputT = decltype(dummy);
      for (int path = 0; path < nPaths; ++path)
        if (!calcCumulativeSignature<InputT>(
                batchPtr(a, 2, path), point_stride, dim_stride, lengthOfPath, d,
                level, eachOutputSize,
                od + path * eachOutputSize * (lengthOfPath - 1)))
          return false;
      return true;
    };
    bool ok = do_interruptible([&] {
      if (use_float32)
        return doCalcCumul(float{});
      else
        return doCalcCumul(double{});
    });
    if (!ok)
      return nullptr;
    return o;
  }
  if (format == 0) {
    o = simpleNew_ownLastDim(ndims - 1, a, eachOutputSize, OutT::typenum);
    if (!o)
      return nullptr;
    out_data = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  } else if (format != 1)
    ERR("Invalid format requested");
  ReleasableRefHolder o_(o);

  Signature s;
  auto doCalcSig = [&](auto dummy) -> bool {
    using InputT = decltype(dummy);
    for (int path = 0; path < nPaths; ++path) {
      if (!calcSignature<InputT>(s, batchPtr(a, 2, path), point_stride,
                                 dim_stride, lengthOfPath, d, level))
        return false;
      if (format == 0)
        s.writeOut(out_data + path * eachOutputSize);
    }
    return true;
  };
  bool ok = do_interruptible([&] {
    if (use_float32)
      return doCalcSig(float{});
    else
      return doCalcSig(double{});
  });
  if (!ok)
    return nullptr;

  if (format == 1) {
    o = PyTuple_New(level);
    for (int m = 0; m < level; ++m) {
      npy_intp dims[] = {(npy_intp)calcSigLevelLength(d, m + 1)};
      PyObject *p = PyArray_SimpleNew(1, dims, OutT::typenum);
      if (!p)
        return nullptr;
      auto ptr = static_cast<OutT::T *>(
          PyArray_DATA(reinterpret_cast<PyArrayObject *>(p)));
      for (auto f : s.m_data[m])
        *(ptr++) = (OutT::T)f;
      PyTuple_SET_ITEM(o, m, p);
    }
  }
  o_.release();
  return o;
}

static PyObject *sigMultCount(PyObject *self, PyObject *args) {
  PyObject *data;
  int level = 0;
  int format;
  if (!PyArg_ParseTuple(args, "Oi|i", &data, &level, &format))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(data, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  if (PyArray_NDIM(a) != 2)
    ERR("data must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, 0);
  const int d = (int)PyArray_DIM(a, 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d < 1)
    ERR("Path must have positive dimension");

  double out = CalcSignature::Signature::concatenateWithMultCount(d, level) *
               (lengthOfPath - 2);
  out += CalcSignature::Signature::sigOfSegmentMultCount(d, level) *
         (lengthOfPath - 1);
  return PyLong_FromDouble(out);
}

// Take a Numpy array which is already ensured contiguous and
// either NPY_FLOAT32 or NPY_FLOAT64
// and read from it as doubles
class ReadArrayAsDoubles {
  vector<double> m_store;
  double *m_ptr = nullptr;

public:
  // returns true on bad_alloc
  bool read(PyArrayObject *a, size_t size) {
    if (PyArray_TYPE(a) == NPY_FLOAT64) {
      m_ptr = static_cast<double *>(PyArray_DATA(a));
      return false;
    }
    try {
      m_store.resize(size);
    } catch (std::bad_alloc &) {
      return true;
    }
    auto source = static_cast<float *>(PyArray_DATA(a));
    for (size_t i = 0; i < size; ++i)
      m_store[i] = source[i];
    m_ptr = m_store.data();
    return false;
  }
  const double *ptr() const { return m_ptr; }
};

static PyObject *sigBackwards(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  int level = 0;
  if (!PyArg_ParseTuple(args, "OOi", &a2, &a1, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  bool is_float32 =
      PyArray_Check(a1) && PyArray_TYPE((PyArrayObject *)a1) == NPY_FLOAT32;
  PyObject *aa = PyArray_FROM_OTF(a1, is_float32 ? NPY_FLOAT32 : NPY_FLOAT64,
                                  NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("path must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  bool use_float32 = (PyArray_TYPE(a) == NPY_FLOAT32);
  PyObject *bb = PyArray_FROM_OTF(a2, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!bb)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;

  int ndims = PyArray_NDIM(a);
  if (ndims < 2)
    ERR("path must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, ndims - 2);
  const int d = (int)PyArray_DIM(a, ndims - 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d < 1)
    ERR("Path must have positive dimension");
  size_t sigLength = calcSigTotalLength(d, level);
  if (PyArray_NDIM(b) != ndims - 1)
    ERR("derivs has the wrong number of dimensions");
  if (sigLength != (size_t)PyArray_DIM(b, ndims - 2))
    ERR(("derivs should have length " + std::to_string(sigLength) +
         " but it has length " + std::to_string(PyArray_DIM(b, ndims - 2)))
            .c_str());
  int nPaths = 1;
  for (int i = 0; i + 2 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between paths and derivatives");
    nPaths *= (int)x;
  }
  size_t eachInputSize = (size_t)(lengthOfPath * d);

  // Path: use strides directly (zero-copy)
  npy_intp *a_strides = PyArray_STRIDES(a);
  npy_intp point_stride = a_strides[ndims - 2];
  npy_intp dim_stride = a_strides[ndims - 1];

  // Derivs: keep fallback for non-contiguous last dim (rare for 1D-inner)
  ReleasableRefHolder b_contig_;
  if (!isInnerContiguous(b, 1)) {
    PyObject *bb2 = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
    if (!bb2)
      ERR("derivs must be (convertible to) a numpy array");
    b_contig_.releaseAndSet(bb2);
    b = (PyArrayObject *)bb2;
  }

  using OutT = UseFloat;
  PyObject *o = PyArray_ZEROS(ndims, PyArray_DIMS(a), OutT::typenum, 0);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  if (use_float32) {
    for (int path = 0; path < nPaths; ++path)
      CalcSignature::sigBackwardsRaw<float>(
          d, level, lengthOfPath, batchPtr(a, 2, path), point_stride,
          dim_stride, (double *)batchPtr(b, 1, path),
          out + path * eachInputSize);
  } else {
    for (int path = 0; path < nPaths; ++path)
      CalcSignature::sigBackwardsRaw<double>(
          d, level, lengthOfPath, batchPtr(a, 2, path), point_stride,
          dim_stride, (double *)batchPtr(b, 1, path),
          out + path * eachInputSize);
  }
  return o;
}

static PyObject *sigJacobian(PyObject *self, PyObject *args) {
  PyObject *a1;
  int level = 0;
  if (!PyArg_ParseTuple(args, "Oi", &a1, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  if (PyArray_NDIM(a) != 2)
    ERR("data must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, 0);
  const int d = (int)PyArray_DIM(a, 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d < 1)
    ERR("Path must have positive dimension");
  size_t sigLength = calcSigTotalLength(d, level);
  ReadArrayAsDoubles input;
  if (input.read(a, lengthOfPath * d))
    ERR("Out of memory");

  npy_intp dims[] = {(npy_intp)lengthOfPath, (npy_intp)d, (npy_intp)sigLength};
  using OutT = UseFloat;
  PyObject *o = PyArray_SimpleNew(3, dims, OutT::typenum);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  TotalDerivativeSignature::sigJacobian(input.ptr(), lengthOfPath, d, level,
                                        out);
  return o;
}

static PyObject *sigJoin(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  int level = 0;
  double fixedLast = std::numeric_limits<double>::quiet_NaN();
  if (!PyArg_ParseTuple(args, "OOi|d", &a1, &a2, &level, &fixedLast))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_FROM_OTF(a1, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_FROM_OTF(a2, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!bb)
    ERR("data must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and data must have the same number of dimensions");
  const int d_given = (int)PyArray_DIM(b, ndims - 1);
  if (d_given < 1)
    ERR("Path must have positive dimension");
  const int d_out = std::isnan(fixedLast) ? d_given : d_given + 1;
  size_t sigLength = calcSigTotalLength(d_out, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("signatures have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and data");
    nPaths *= (int)x;
  }

  ReleasableRefHolder a_contig_, b_contig_;
  if (!isInnerContiguous(a, 1)) {
    PyObject *aa2 = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
    if (!aa2)
      ERR("sigs must be (convertible to) a numpy array");
    a_contig_.releaseAndSet(aa2);
    a = (PyArrayObject *)aa2;
  }
  if (!isInnerContiguous(b, 1)) {
    PyObject *bb2 = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
    if (!bb2)
      ERR("data must be (convertible to) a numpy array");
    b_contig_.releaseAndSet(bb2);
    b = (PyArrayObject *)bb2;
  }

  ReadArrayAsDoubles sig, displacement;
  if (sig.read(a, nPaths * sigLength) || displacement.read(b, nPaths * d_given))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  for (int iPath = 0; iPath < nPaths; ++iPath)
    CalcSignature::sigJoin(d_out, level, (double *)batchPtr(a, 1, iPath),
                           (double *)batchPtr(b, 1, iPath), fixedLast,
                           out + iPath * sigLength);
  return o;
}

static PyObject *sigJoinBackwards(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  PyObject *a3;
  int level = 0;
  double fixedLast = std::numeric_limits<double>::quiet_NaN();
  if (!PyArg_ParseTuple(args, "OOOi|d", &a3, &a1, &a2, &level, &fixedLast))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("new data must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;
  PyObject *cc = PyArray_ContiguousFromAny(a3, NPY_FLOAT64, 0, 0);
  if (!cc)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder c_(cc);
  PyArrayObject *c = (PyArrayObject *)cc;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and data must have the same number of dimensions");
  if (PyArray_NDIM(c) != ndims)
    ERR("sigs and derivs must have the same number of dimensions");
  const int d_given = (int)PyArray_DIM(b, ndims - 1);
  if (d_given < 1)
    ERR("Path must have positive dimension");
  const int d_out = std::isnan(fixedLast) ? d_given : d_given + 1;
  size_t sigLength = calcSigTotalLength(d_out, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("signatures have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and data");
    if (PyArray_DIM(c, i) != x)
      ERR("mismatched dimensions between sigs and derivs");
    nPaths *= (int)x;
  }
  ReadArrayAsDoubles sig, displacement, derivs;
  if (sig.read(a, nPaths * sigLength) ||
      displacement.read(b, nPaths * d_given) ||
      derivs.read(c, nPaths * sigLength))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o1 = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o1)
    return nullptr;
  PyObject *o2 = PyArray_SimpleNew(ndims, PyArray_DIMS(b), OutT::typenum);
  if (!o2) {
    Py_DECREF(o1);
    return nullptr;
  }

  auto out1 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o1)));
  auto out2 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o2)));
  double dFixedLast = 0;
  for (int iPath = 0; iPath < nPaths; ++iPath)
    CalcSignature::sigJoinBackwards(d_out, level, sig.ptr() + iPath * sigLength,
                                    displacement.ptr() + iPath * d_given,
                                    derivs.ptr() + iPath * sigLength, fixedLast,
                                    out1 + iPath * sigLength,
                                    out2 + iPath * d_given, dFixedLast);
  // PyTuple_Pack doesn't steal references
  if (std::isnan(fixedLast))
    return Py_BuildValue("(NN)", o1, o2);
  else
    return Py_BuildValue("(NNd)", o1, o2, dFixedLast);
}

static PyObject *sigCombine(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  int d = 0;
  int level = 0;
  if (!PyArg_ParseTuple(args, "OOii", &a1, &a2, &d, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  if (d < 1)
    ERR("dimension must be positive");
  PyObject *aa = PyArray_FROM_OTF(a1, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("first signature must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_FROM_OTF(a2, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!bb)
    ERR("second signature must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("signatures inputs must have the same number of dimensions");
  size_t sigLength = calcSigTotalLength(d, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("first input has unexpected length");
  if (sigLength != (size_t)PyArray_DIM(b, ndims - 1))
    ERR("second input has unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between inputs");
    nPaths *= (int)x;
  }

  ReleasableRefHolder a_contig_, b_contig_;
  if (!isInnerContiguous(a, 1)) {
    PyObject *aa2 = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
    if (!aa2)
      ERR("first signature must be (convertible to) a numpy array");
    a_contig_.releaseAndSet(aa2);
    a = (PyArrayObject *)aa2;
  }
  if (!isInnerContiguous(b, 1)) {
    PyObject *bb2 = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
    if (!bb2)
      ERR("second signature must be (convertible to) a numpy array");
    b_contig_.releaseAndSet(bb2);
    b = (PyArrayObject *)bb2;
  }

  ReadArrayAsDoubles sig1, sig2;
  if (sig1.read(a, nPaths * sigLength) || sig2.read(b, nPaths * sigLength))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  CalcSignature::SigCombiner sc;
  for (int iPath = 0; iPath < nPaths; ++iPath)
    sc.sigCombine(d, level, (double *)batchPtr(a, 1, iPath),
                  (double *)batchPtr(b, 1, iPath), out + iPath * sigLength);
  return o;
}

static PyObject *sigCombineBackwards(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  PyObject *a3;
  int d = 0;
  int level = 0;
  if (!PyArg_ParseTuple(args, "OOOii", &a3, &a1, &a2, &d, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  if (d < 1)
    ERR("dimension must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("first signature must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("second signature must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;
  PyObject *cc = PyArray_ContiguousFromAny(a3, NPY_FLOAT64, 0, 0);
  if (!cc)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder c_(cc);
  PyArrayObject *c = (PyArrayObject *)cc;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("signature inputs must have the same number of dimensions");
  if (PyArray_NDIM(c) != ndims)
    ERR("sigs and derivs must have the same number of dimensions");
  size_t sigLength = calcSigTotalLength(d, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("first input has unexpected length");
  if (sigLength != (size_t)PyArray_DIM(b, ndims - 1))
    ERR("second input has unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between inputs");
    if (PyArray_DIM(c, i) != x)
      ERR("mismatched dimensions between sigs and derivs");
    nPaths *= (int)x;
  }

  ReadArrayAsDoubles sig1, sig2, derivs;
  if (sig1.read(a, nPaths * sigLength) || sig2.read(b, nPaths * sigLength) ||
      derivs.read(c, nPaths * sigLength))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o1 = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o1)
    return nullptr;
  PyObject *o2 = PyArray_SimpleNew(ndims, PyArray_DIMS(b), OutT::typenum);
  if (!o2) {
    Py_DECREF(o1);
    return nullptr;
  }

  auto out1 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o1)));
  auto out2 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o2)));
  CalcSignature::SigCombiner sc;
  for (int iPath = 0; iPath < nPaths; ++iPath)
    sc.sigCombineBackwards(d, level, sig1.ptr() + iPath * sigLength,
                           sig2.ptr() + iPath * sigLength,
                           derivs.ptr() + iPath * sigLength,
                           out1 + iPath * sigLength, out2 + iPath * sigLength);
  return Py_BuildValue("(NN)", o1, o2);
}

static PyObject *sigScale(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  int level = 0;
  if (!PyArg_ParseTuple(args, "OOi", &a1, &a2, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("scales must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and scales must have the same number of dimensions");
  const int d = (int)PyArray_DIM(b, ndims - 1);
  if (d < 1)
    ERR("scales must have positive dimension");
  size_t sigLength = calcSigTotalLength(d, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("signatures have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and scales");
    nPaths *= (int)x;
  }

  ReadArrayAsDoubles sig, scales;
  if (sig.read(a, nPaths * sigLength) || scales.read(b, nPaths * d))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  for (int iPath = 0; iPath < nPaths; ++iPath)
    CalcSignature::sigScale(d, level, sig.ptr() + iPath * sigLength,
                            scales.ptr() + iPath * d, out + iPath * sigLength);
  return o;
}

static PyObject *sigScaleBackwards(PyObject *self, PyObject *args) {
  PyObject *a1;
  PyObject *a2;
  PyObject *a3;
  int level = 0;
  if (!PyArg_ParseTuple(args, "OOOi", &a3, &a1, &a2, &level))
    return nullptr;
  if (level < 1)
    ERR("level must be positive");
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("scales must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;
  PyObject *cc = PyArray_ContiguousFromAny(a3, NPY_FLOAT64, 0, 0);
  if (!cc)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder c_(cc);
  PyArrayObject *c = (PyArrayObject *)cc;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and data must have the same number of dimensions");
  if (PyArray_NDIM(c) != ndims)
    ERR("sigs and derivs must have the same number of dimensions");
  const int d = (int)PyArray_DIM(b, ndims - 1);
  if (d < 1)
    ERR("Path must have positive dimension");
  size_t sigLength = calcSigTotalLength(d, level);
  if (sigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("signatures have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and data");
    if (PyArray_DIM(c, i) != x)
      ERR("mismatched dimensions between sigs and derivs");
    nPaths *= (int)x;
  }

  ReadArrayAsDoubles sig, scale, derivs;
  if (sig.read(a, nPaths * sigLength) || scale.read(b, nPaths * d) ||
      derivs.read(c, nPaths * sigLength))
    ERR("Out of memory");

  using OutT = UseFloat;
  PyObject *o1 = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o1)
    return nullptr;
  PyObject *o2 = PyArray_SimpleNew(ndims, PyArray_DIMS(b), OutT::typenum);
  if (!o2) {
    Py_DECREF(o1);
    return nullptr;
  }

  auto out1 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o1)));
  auto out2 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o2)));
  for (int iPath = 0; iPath < nPaths; ++iPath)
    CalcSignature::sigScaleBackwards(
        d, level, sig.ptr() + iPath * sigLength, scale.ptr() + iPath * d,
        derivs.ptr() + iPath * sigLength, out1 + iPath * sigLength,
        out2 + iPath * d);
  return Py_BuildValue("(NN)", o1, o2);
}
#endif // IISIGNATURE_NO_NUMPY

const char *const logSigFunction_id = "iisignature.LogSigFunction";
LogSigFunction *getLogSigFunction(PyObject *p) {
  if (!PyCapsule_IsValid(p, logSigFunction_id))
    ERR("I have received an input which is not from iisignature.prepare()");
  return (LogSigFunction *)PyCapsule_GetPointer(p, logSigFunction_id);
}

static void killLogSigFunction(PyObject *p) { delete getLogSigFunction(p); }

// returns true on success
bool getData() {
  if (g_bchLyndon20_dat) // already set
    return true;
  PyObject *get_data = nullptr;
  // PyObject* pkgutil = PyImport_AddModule("pkgutil"); //flaky, returns
  // borrowed
  PyObject *pkgutil = PyImport_ImportModule("pkgutil");
  if (pkgutil) {
    RefHolder p_(pkgutil);
    get_data = PyObject_GetAttrString(pkgutil, "get_data");
  }
  if (!get_data) {
    PyObject *pkgutil = PyImport_ImportModule("pkg_resources");
    if (pkgutil) {
      RefHolder p_(pkgutil);
      get_data = PyObject_GetAttrString(pkgutil, "resource_string");
    }
  }
  if (!get_data)
    ERRb("neither pkgutil nor pkg_resources is working");
  RefHolder get_data_(get_data);
  PyObject *ii = PyUnicode_FromString("iisignature_data");
  RefHolder ii_(ii);
  PyObject *name = PyUnicode_FromString("bchLyndon20.dat");
  RefHolder name_(name);
  PyObject *o = PyObject_CallFunctionObjArgs(get_data, ii, name, NULL);
  if (!o)
    return false;
  if (o == Py_None)
    ERRb("Cannot find iisignature_data module");
  if (!PyBytes_CheckExact(o)) {
    // std::cerr<<PyBytes_AsString(PyObject_Bytes(o))<<"\n";
    ERRb("unexpected type from pkgutil.get_data");
  }
  g_bchLyndon20_dat = PyBytes_AsString(o);
  // deliberately leak a reference to o - we'll keep it forever.
  return true;
}

#ifndef IISIGNATURE_NO_NUMPY
// this class just provides access to the functions lstsq, pinv, svd and
// transpose from numpy
class LeastSquares {
  PyObject *m_transpose;
  PyObject *m_lstsq;
  PyObject *m_pinv;
  PyObject *m_svd;
  PyObject *m_qrScipy = nullptr;
  ReleasableRefHolder m_t_, m_l_, m_p_, m_s_, m_q_; // we can do better than
                                                    // this
  mutable vector<float> m_tempSpace;

public:
  bool m_ok = false;
  explicit LeastSquares(bool wantQR = false) {
    PyObject *numpy = PyImport_AddModule("numpy");
    if (!numpy)
      return;
    PyObject *linalg = PyObject_GetAttrString(numpy, "linalg");
    if (!linalg)
      return;
    RefHolder linalg_(linalg);
    m_transpose = PyObject_GetAttrString(numpy, "transpose");
    if (!m_transpose)
      return;
    m_t_.releaseAndSet(m_transpose);
    m_lstsq = PyObject_GetAttrString(linalg, "lstsq");
    if (!m_lstsq)
      return;
    m_l_.releaseAndSet(m_lstsq);
    m_pinv = PyObject_GetAttrString(linalg, "pinv");
    if (!m_pinv)
      return;
    m_p_.releaseAndSet(m_pinv);
    m_svd = PyObject_GetAttrString(linalg, "svd");
    if (!m_svd)
      return;
    m_s_.releaseAndSet(m_svd);
    m_ok = true;

    if (wantQR) {
      // For some reason I don't understand, I have to use the higher level
      // PyImport_Import to load scipy.linalg
      auto pckg_name = PyUnicode_FromString("scipy.linalg");
      RefHolder pckg_name_(pckg_name);
      PyObject *scipyLinalg = PyImport_Import(pckg_name);
      if (!scipyLinalg) {
        PyErr_Clear();
        return;
      }
      RefHolder scipyLinalg_(scipyLinalg);
      m_qrScipy = PyObject_GetAttrString(scipyLinalg, "qr");
      if (m_qrScipy)
        m_q_.releaseAndSet(m_qrScipy);
      else
        PyErr_Clear();
    }
  }
  PyObject *lstsqWithTranspose(PyObject *a1, PyObject *a2) const {
    PyObject *a1t = PyObject_CallFunctionObjArgs(m_transpose, a1, NULL);
    if (!a1t)
      return nullptr;
    RefHolder a1t_(a1t);
    PyObject *o = PyObject_CallFunctionObjArgs(m_lstsq, a1t, a2, NULL);
    if (!o)
      return nullptr;
    // return o;
    // return PyTuple_Pack(2,o,a1);
    RefHolder o_(o);
    PyObject *answer = PyTuple_GetItem(o, 0);
    Py_INCREF(answer);
    return answer;
  }
  PyObject *lstsqNoTranspose(PyObject *a1, PyObject *a2) const {
    PyObject *o = PyObject_CallFunctionObjArgs(m_lstsq, a1, a2, NULL);
    if (!o)
      return nullptr;
    // return o;
    // return PyTuple_Pack(2,o,a1);
    RefHolder o_(o);
    PyObject *answer = PyTuple_GetItem(o, 0);
    Py_INCREF(answer);
    return answer;
  }
  // a numpy matrix of an existing vector. Const correctness is user's
  // responsibility
  template <class Type> class MatrixOfVector {
  public:
    PyObject *m_mat;
    MatrixOfVector(const vector<typename Type::T> &v, npy_intp r, npy_intp c) {
      npy_intp dims[] = {r, c};
      m_mat =
          PyArray_SimpleNewFromData(2, dims, Type::typenum, (void *)v.data());
    }
    MatrixOfVector(const MatrixOfVector &) = delete;
    MatrixOfVector operator=(const MatrixOfVector &) = delete;
    ~MatrixOfVector() { Py_DECREF(m_mat); }
  };

  // given an rxc matrix mat, find its rank and the first part of its SVD
  // on failure, m_ok will be false.
  // expect r>c, so that we are interested in the span of the columns,
  class SVD {
  public:
    double *m_u; // is rxr if full_matrices, else rxc
    bool m_ok = false;
    ReleasableRefHolder m_delete;
    int m_rank = 0;
    SVD(const vector<double> &mat, npy_intp r, npy_intp c, bool full_matrices,
        LeastSquares &ls) {
      MatrixOfVector<UseDouble> mat_(mat, r, c);
      if (mat.size() != ((size_t)r * c))
        ERRr("bad use of SVD");
#if 0
      for (int rr = 0, i = 0; rr < r; ++rr) {
        for (int cc = 0; cc < c; ++cc, ++i)
          std::cout << mat[i] << ",";
        std::cout << "\n";
      }
#endif
      PyObject *svd = PyObject_CallFunctionObjArgs(
          ls.m_svd, mat_.m_mat, full_matrices ? Py_True : Py_False, NULL);
      if (!svd)
        return;
      m_delete.releaseAndSet(svd);
      PyObject *svs = PyTuple_GetItem(svd, 1);
      PyArrayObject *svd0 = (PyArrayObject *)PyTuple_GetItem(svd, 0);
      bool ok = PyArray_NDIM((PyArrayObject *)svs) == 1 &&
                PyArray_TYPE((PyArrayObject *)svs) == NPY_FLOAT64 &&
                PyArray_NDIM(svd0) == 2 && PyArray_TYPE(svd0) == NPY_FLOAT64 &&
                PyArray_DIM(svd0, 0) == r &&
                PyArray_DIM(svd0, 1) == (full_matrices ? r : c) &&
                PyArray_ISCARRAY_RO(svd0) &&
                PyArray_ISCARRAY_RO((PyArrayObject *)svs);
      if (!ok)
        ERRr("numpy svd returned something strange.");
      npy_intp n_svs = PyArray_DIM((PyArrayObject *)svs, 0);
#ifdef PRINT_SVS
      std::cout << "\n";
#endif
      for (npy_intp i = 0; i < n_svs; ++i) {
        double sv = *((double *)PyArray_GETPTR1((PyArrayObject *)svs, i));
#ifdef PRINT_SVS
        std::cout << sv << ",";
#endif
        if (sv > 0.000001)
          ++m_rank;
      }
#ifdef PRINT_SVS
      std::cout << std::endl;
#endif
      m_u = (double *)PyArray_DATA(svd0);
      m_ok = true;
    }
  };

  // given an rxc matrix mat, find its rank and the Q of its QR
  // on failure, m_ok will be false.
  // We have r>c, and we are interested in the span of the columns.
  // So we need to use scipy's qr, which provides pivoting.
  // The user of this class must have already checked
  // that m_qrScipy is not nullptr.
  class QR {
  public:
    double *m_q; // is rxc
    bool m_ok = false;
    ReleasableRefHolder m_delete;
    int m_rank = 0;
    QR(const vector<double> &mat, npy_intp r, npy_intp c, LeastSquares &ls) {
      MatrixOfVector<UseDouble> mat_(mat, r, c);
      if (mat.size() != ((size_t)r * c))
        ERRr("bad use of QR");
      PyObject *qr = PyObject_CallFunction(ls.m_qrScipy, "Oissi", mat_.m_mat, 0,
                                           NULL, "economic", 1);
      if (!qr)
        return;
      m_delete.releaseAndSet(qr);
      bool ok1 = PyTuple_CheckExact(qr) && PyTuple_Size(qr) > 1 &&
                 PyArray_Check(PyTuple_GetItem(qr, 0)) &&
                 PyArray_Check(PyTuple_GetItem(qr, 1));
      if (!ok1)
        ERRr("scipy qr returned something very strange.");
      PyArrayObject *qr0 = (PyArrayObject *)PyTuple_GetItem(qr, 0);
      PyArrayObject *qr1 = (PyArrayObject *)PyTuple_GetItem(qr, 1);
      // NB: It seems scipy returns qr0 as fortran, numpy's qr doesnt
      bool fortran = PyArray_ISFARRAY_RO(qr0);
      bool qr0contig = fortran;
      bool ok2 = PyArray_NDIM(qr0) == 2 && PyArray_TYPE(qr0) == NPY_FLOAT64 &&
                 PyArray_DIM(qr0, 0) == r && PyArray_DIM(qr0, 1) == c &&
                 PyArray_NDIM(qr1) == 2 && PyArray_TYPE(qr1) == NPY_FLOAT64 &&
                 PyArray_DIM(qr1, 0) == c && PyArray_DIM(qr1, 1) == c &&
                 qr0contig;
      if (!ok2)
        ERRr("scipy qr returned something strange.");
#ifdef PRINT_SVS
      std::cout << "\n";
#endif
      for (npy_intp i = 0; i < c; ++i) {
        double sv = *((double *)PyArray_GETPTR2((PyArrayObject *)qr1, i, i));
#ifdef PRINT_SVS
        std::cout << sv << ",";
#endif
        if (sv > 0.000001 || sv < -0.000001)
          ++m_rank;
      }
#ifdef PRINT_SVS
      std::cout << std::endl;
#endif
      m_q = (double *)PyArray_DATA(qr0);
      m_ok = true;
    }
  };

  // this replaces the r x c matrix matrix by its Moore-Penrose pseudoinverse,
  // which is a c x r matrix
  // returns true on success
  bool inplacePinvMatrix(float *matrix, size_t r, size_t c) const {
    m_tempSpace.assign(matrix, matrix + r * c);
    npy_intp dims2[] = {(npy_intp)(r), (npy_intp)c};
    PyObject *mat =
        PyArray_SimpleNewFromData(2, dims2, NPY_FLOAT32, m_tempSpace.data());
    RefHolder m_(mat);
    PyObject *o = PyObject_CallFunctionObjArgs(m_pinv, mat, NULL);
    if (!o)
      return false;
    RefHolder o_(o);
    PyArrayObject *oa = reinterpret_cast<PyArrayObject *>(o);
    bool ok = PyArray_Check(o) && PyArray_TYPE(oa) == NPY_FLOAT32 &&
              PyArray_NDIM(oa) == 2 && PyArray_DIM(oa, 0) == ((npy_intp)c) &&
              PyArray_DIM(oa, 1) == ((npy_intp)r);
    if (!ok)
      ERRb("bad output from pinv");
    PyArrayObject *outc = PyArray_GETCONTIGUOUS(oa);
    RefHolder l2_(reinterpret_cast<PyObject *>(outc));
    float *ptr = static_cast<float *>(PyArray_DATA(outc));
    for (size_t i = 0; i < r * c; ++i)
      matrix[i] = ptr[i];
    return true;
  }

  /*
     //inp is a dxm matrix and sub is a dxn matrix with n<m
    //set out to a dx(roughly m-n) matrix whose columns are perp to colspan(sub)
    and each other
    //s.t. colspan(out)=colspan(inp)
    //This method may be a big waste of time. We shouldn't need d^2 space.
    //I wonder if qr would beat svd, but I think we'd still need two.
    bool projectAwayFrom1(const vector<double>& inp, const vector<double>& sub,
    npy_intp d, vector<double>& out) { npy_intp m = inp.size() / d; npy_intp n =
    sub.size() / d;

      SVD svd(sub, d, n, true, *this);
      if (!svd.m_ok)
        return false;

      double* s1 = svd.m_u;
      int sub_rank = svd.m_rank;
      vector<double> inp_projected(d*m, 0);
      //std::cout << "!"<< d << "," << m << "," << n << "," << sub_rank <<
    std::endl;
      //s1 is a dxd array. If s2 is s1 without its first sub_rank columns,
      //then s3=s2 s2^T is a projection matrix away from sub
      //we want inp_projected= s3.inp = s2 . (s2^T.inp) =: s2.s4

      if (0) {
        vector<double> s3(d*d, 0);
        for (int i = 0; i < d; ++i) {
          for (int j = 0; j + sub_rank < d; ++j) {
            for (int k = 0; k < d; ++k) {
              s3.at(i*d + k) += s1[i*d + sub_rank + j] * s1[k*d + sub_rank + j];
            }
          }
        }

        for (int i = 0; i < m; ++i)
          for (int j = 0; j < d; ++j)
            for (int k = 0; k < d; ++k) {
              //if (i*m + k >= inp.size())
                //std::cout << "'" <<
    inp.size()<<","<<s3.size()<<","<<inp_projected.size()
                //  << "," << m << "," << d << "," << i << "," << j << "," << k
    << std::endl; inp_projected.at(j*m + i) += inp.at(k*m + i) * s3.at(j*d + k);
            }
      }
      else { // group s.t. second multiply happens first
        npy_intp s4_rows = d - sub_rank;
        vector<double> s4((d - sub_rank)*d);
        for (npy_intp i = 0; i < d; ++i)
          for (npy_intp j = 0; j < m; ++j)
            for (npy_intp k = 0; k < s4_rows; ++k)
              s4.at(k*s4_rows + j) += s1[i*d + sub_rank + k] * inp.at(i*m+j);
        std::cout << "p4" << std::endl;
        for (npy_intp i = 0; i < d; ++i)
          for (npy_intp j = 0; j < m; ++j)
            for (npy_intp k = 0; k < s4_rows; ++k)
              inp_projected.at(i*m + j) += s1[i*d + sub_rank + k] *
    s4.at(k*s4_rows + j);
      }

      SVD svd2(inp_projected, d, m, false, *this);
      if (!svd2.m_ok)
        return false;

      int out_rank = svd2.m_rank;
      out.assign(d*out_rank,0);
      for (npy_intp i = 0; i < d; ++i)
        for (int j = 0; j < out_rank; ++j) {
          //std::cout << i << "," << j << "," << d << "," << out_rank <<
    std::endl; out.at(i*out_rank + j) = svd2.m_u[i*m + j];
        }
      return true;
    }
   */

  // inp is a dxm matrix and sub is a dxn matrix with n<m
  // set out to a dx(roughly m-n) matrix whose columns are perp to colspan(sub)
  // and each other s.t. colspan(out)=colspan(inp) This is like the previous
  // only the projection matrix is not constructed. space d^2 is never needed.
  bool projectAwayFromSVD(const vector<double> &inp, const vector<double> &sub,
                          npy_intp d, vector<double> &out) {
    npy_intp m = inp.size() / d;
    npy_intp n = sub.size() / d;

    SVD svd(sub, d, n, false, *this);
    if (!svd.m_ok)
      return false;

    vector<double> inp_projected = inp;
    for (npy_intp i = 0; i < m; ++i)
      for (npy_intp j = 0; j < svd.m_rank; ++j) {
        double dot_product = 0;
        for (npy_intp k = 0; k < d; ++k)
          dot_product += inp.at(k * m + i) * svd.m_u[k * n + j];
        for (npy_intp k = 0; k < d; ++k)
          inp_projected[k * m + i] -= dot_product * svd.m_u[k * n + j];
      }

    SVD svd2(inp_projected, d, m, false, *this);
    if (!svd2.m_ok)
      return false;

    int out_rank = svd2.m_rank;
    out.assign(d * out_rank, 0);
    for (int i = 0; i < d; ++i)
      for (int j = 0; j < out_rank; ++j) {
        // std::cout << i << "," << j << "," << d << "," << out_rank <<
        // std::endl;
        out.at(i * out_rank + j) = svd2.m_u[i * m + j];
      }
    return true;
  }
  // Like projectAwayFromSVD only using QR instead of SVD
  //  - relies on scipy
  bool projectAwayFromQR(const vector<double> &inp, const vector<double> &sub,
                         npy_intp d, vector<double> &out) {
    npy_intp m = inp.size() / d;
    npy_intp n = sub.size() / d;

    if (!m_qrScipy)
      ERRb("scipy not available, QR method cannot be used");

    QR qr1(sub, d, n, *this);
    if (!qr1.m_ok)
      return false;

    vector<double> inp_projected = inp;
    for (npy_intp i = 0; i < m; ++i)
      for (npy_intp j = 0; j < qr1.m_rank; ++j) {
        double dot_product = 0;
        for (npy_intp k = 0; k < d; ++k)
          dot_product += inp.at(k * m + i) * qr1.m_q[k + j * d];
        for (npy_intp k = 0; k < d; ++k)
          inp_projected[k * m + i] -= dot_product * qr1.m_q[k + j * d];
      }

    QR qr2(inp_projected, d, m, *this);
    if (!qr2.m_ok)
      return false;

    int out_rank = qr2.m_rank;
    out.assign(d * out_rank, 0);
    for (int i = 0; i < d; ++i)
      for (int j = 0; j < out_rank; ++j) {
        // std::cout << i << "," << j << "," << d << "," << out_rank <<
        // std::endl;
        out.at(i * out_rank + j) = qr2.m_q[i + j * d];
      }
    return true;
  }
};

// things which have to be done at the end of prepare, relying on Numpy
// returns false on error
// can also be called in logsigbackwards
bool postPrepare(LogSigFunction *lsf) {
  LeastSquares ls;
  if (!ls.m_ok)
    return false;
  for (auto &i : lsf->m_smallSVDs)
    for (auto &j : i) {
      if (j.m_matrix.empty())
        continue;
      bool ok = ls.inplacePinvMatrix(j.m_matrix.data(), j.m_sources.size(),
                                     j.m_dests.size());
      if (!ok)
        return false;
    }
  return true;
}
#endif // IISIGNATURE_NO_NUMPY

// This function takes a dim1xdim2 matrix and an rhs and calls resultAction on a
// pointer to the results of lstsq on it returns true on success
/*
template<typename T>
bool callLeastSquares(LeastSquares& ls, float* matrix, size_t dim1, size_t dim2,
float* rhs, bool transpose, T&& resultAction) { const size_t rhs_length =
(transpose ? dim2 : dim1);
  //const size_t out_length = (transpose ? dim1 : dim2);
  npy_intp dims[] = { (npy_intp)rhs_length };
  PyObject* rhs_ = PyArray_SimpleNewFromData(1, dims, NPY_FLOAT32, rhs);
  RefHolder r_(rhs_);

  npy_intp dims2[] = { (npy_intp)(dim1), (npy_intp)dim2 };
  PyObject* mat = PyArray_SimpleNewFromData(2, dims2, NPY_FLOAT32, matrix);
  RefHolder m_(mat);
  PyObject* out = transpose ? ls.lstsqWithTranspose(mat, rhs_) :
ls.lstsqNoTranspose(mat, rhs_); if (!out) return false; RefHolder o_(out); auto
outa = reinterpret_cast<PyArrayObject*>(out); PyArrayObject* outc =
PyArray_GETCONTIGUOUS(outa); RefHolder l2_(reinterpret_cast<PyObject*>(outc));
  float* ptr = static_cast<float*>(PyArray_DATA(outc));
  if (false) {
    for (size_t row = 0;row < dim1;row++) {
      for (size_t col = 0; col < dim2; ++col)
        std::cout << matrix[row*dim2 + col] << ",";
      std::cout << "\n";
    }
    std::cout << (transpose ? "transpose times\n" : "times\n");
    for (size_t row = 0; row < out_length;++row)
      std::cout << ptr[row] << "\n";
    std::cout << "is\n";
    for (size_t row = 0; row<rhs_length;++row)
      std::cout <<rhs[row] << "\n";
    std::cout << std::endl;
  }
  resultAction(ptr);
  return true;
}
*/

static PyObject *prepare(PyObject *self, PyObject *args) {
  int level = 0, dim = 0;
  const char *methods = nullptr;
  if (!PyArg_ParseTuple(args, "ii|z", &dim, &level, &methods))
    return nullptr;
  if (!getData())
    return nullptr;
  if (dim < 2)
    ERR("dimension must be at least 2");
  if (level < 1)
    ERR("level must be positive");
  WantedMethods wantedmethods;
  std::string methodString;
  if (methods)
    methodString = methods;
  if (setWantedMethods(wantedmethods, dim, level, false, false, methodString))
    ERR(wantedmethods.m_errMsg);
  auto basis = wantedmethods.m_want_matchCoropa ? LieBasis::StandardHall
                                                : LieBasis::Lyndon;
  std::unique_ptr<LogSigFunction> lsf(new LogSigFunction(basis));
  bool ok = do_interruptible_releasing_lock([&] {
    makeLogSigFunction(dim, level, *lsf, wantedmethods, interrupt);
    return true;
  });
  if (!ok)
    return nullptr;

#ifndef IISIGNATURE_NO_NUMPY
  if (!postPrepare(lsf.get()))
    return nullptr;
#endif // IISIGNATURE_NO_NUMPY

  PyObject *out =
      PyCapsule_New(lsf.release(), logSigFunction_id, killLogSigFunction);
  return out;
}

static PyObject *basis(PyObject *self, PyObject *args) {
  PyObject *c;
  if (!PyArg_ParseTuple(args, "O", &c))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(c);
  if (!lsf)
    return nullptr;
  auto &elementList = lsf->m_basisElements;
  if (elementList.empty())
    ERR("dim too large to list basis");
  PyObject *o = PyTuple_New(elementList.size());
  if (!o)
    return nullptr;
  for (size_t i = 0; i < elementList.size(); ++i) {
    std::ostringstream oss;
    printBasisEltBracketsDigits(*elementList[i], oss);
    std::string s = oss.str();
    PyTuple_SET_ITEM(o, i, PyUnicode_FromString(s.c_str()));
  }
  return o;
}

static PyObject *info(PyObject *self, PyObject *args) {
  PyObject *c;
  if (!PyArg_ParseTuple(args, "O", &c))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(c);
  if (!lsf)
    return nullptr;
  std::string methods;
  if (lsf->m_f)
    methods += 'C';
  if (!lsf->m_fd.m_formingT.empty())
    methods += 'O';
  if (lsf->canProjectToBasis())
    methods += 'S';
  if (lsf->canCalculateViaArea())
    methods += 'A';
  methods += 'X';
  PyObject *logsig2sig = lsf->m_support_logsig2sig ? Py_True : Py_False;
  const char *basis =
      (lsf->m_s.m_basis == LieBasis::StandardHall) ? "Standard Hall" : "Lyndon";
  return Py_BuildValue("{sisisssssO}", "dimension", lsf->m_dim, "level",
                       lsf->m_level, "methods", methods.c_str(), "basis", basis,
                       "logsigtosig_supported", logsig2sig);
}

static int getSigLength(LogSigFunction *lsf) {
  if (lsf->m_siglength != 0)
    return lsf->m_siglength;
  return calcSigTotalLength(lsf->m_dim, lsf->m_level);
}

#ifndef IISIGNATURE_NO_NUMPY
static PyObject *logsig(PyObject *self, PyObject *args) {
  PyObject *a1, *a2;
  const char *methods = nullptr;
  if (!PyArg_ParseTuple(args, "OO|z", &a1, &a2, &methods))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a2);
  if (!lsf)
    return nullptr;
  WantedMethods wantedmethods;
  std::string methodString;
  if (methods)
    methodString = methods;
  if (setWantedMethods(wantedmethods, lsf->m_dim, lsf->m_level, true, false,
                       methodString))
    ERR(wantedmethods.m_errMsg);
  bool is_float32 =
      PyArray_Check(a1) && PyArray_TYPE((PyArrayObject *)a1) == NPY_FLOAT32;
  PyObject *aa = PyArray_FROM_OTF(a1, is_float32 ? NPY_FLOAT32 : NPY_FLOAT64,
                                  NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  bool use_float32 = (PyArray_TYPE(a) == NPY_FLOAT32);

  int ndims = PyArray_NDIM(a);
  if (ndims < 2)
    ERR("data must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, ndims - 2);
  const int d = (int)PyArray_DIM(a, ndims - 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d != lsf->m_dim)
    ERR(("Path has dimension " + std::to_string(d) +
         " but we prepared for dimension " + std::to_string(lsf->m_dim))
            .c_str());
  size_t logsiglength = lsf->logSigLength();

  int nPaths = 1;
  for (int i = 0; i + 2 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    nPaths *= (int)x;
  }

  npy_intp *strides = PyArray_STRIDES(a);
  npy_intp point_stride = strides[ndims - 2];
  npy_intp dim_stride = strides[ndims - 1];

  FunctionRunner *f = lsf->m_f.get();
  using OutT = UseDouble;
  if ((wantedmethods.m_compiled_bch && f != nullptr) ||
      (wantedmethods.m_simple_bch && lsf->m_fd.m_length_of_b > 0)) {
    vector<double> displacement(d);
    const bool useCompiled = (f != nullptr && wantedmethods.m_compiled_bch);

    PyObject *o =
        simpleNew_ownLastDim(ndims - 1, a, logsiglength, OutT::typenum);
    if (!o)
      return nullptr;
    auto dest = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
    vector<double> out;
    auto doBCH = [&](auto dummy) {
      using InputT = decltype(dummy);
      for (int path = 0; path < nPaths; ++path) {
        char *data = batchPtr(a, 2, path);
        out.assign(logsiglength, 0);
        if (lengthOfPath > 1) {
          for (int j = 0; j < d; ++j)
            out[j] = *(InputT *)(data + 1 * point_stride + j * dim_stride) -
                     *(InputT *)(data + j * dim_stride);
        }
        for (int i = 2; i < lengthOfPath; ++i) {
          for (int j = 0; j < d; ++j)
            displacement[j] =
                *(InputT *)(data + i * point_stride + j * dim_stride) -
                *(InputT *)(data + (i - 1) * point_stride + j * dim_stride);
          if (useCompiled)
            f->go(out.data(), displacement.data());
          else
            slowExplicitFunction(out.data(), displacement.data(), lsf->m_fd);
        }
        for (double d : out)
          *dest++ = (OutT::T)d;
      }
    };
    if (use_float32)
      doBCH(float{});
    else
      doBCH(double{});
    return o;
  }
  if (wantedmethods.m_expanded ||
      (wantedmethods.m_log_of_signature && lsf->canProjectToBasis())) {
    size_t eachOutputSize = wantedmethods.m_expanded ? (size_t)getSigLength(lsf)
                                                     : (npy_intp)logsiglength;
    PyObject *o =
        simpleNew_ownLastDim(ndims - 1, a, eachOutputSize, OutT::typenum);
    if (!o)
      return nullptr;
    auto outp = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
    ReleasableRefHolder o_(o);

    Signature sig;
    auto doSX = [&](auto dummy) -> bool {
      using InputT = decltype(dummy);
      for (int path = 0; path < nPaths; ++path) {
        char *data = batchPtr(a, 2, path);
        if (!calcSignature<InputT>(sig, data, point_stride, dim_stride,
                                   lengthOfPath, d, lsf->m_level))
          return false;
        logTensorHorner(sig);
        if (wantedmethods.m_expanded)
          sig.writeOut(outp + path * eachOutputSize);
        else {
          projectExpandedLogSigToBasis(outp + path * eachOutputSize, lsf, sig);
        }
      }
      return true;
    };
    bool ok = do_interruptible([&] {
      if (use_float32)
        return doSX(float{});
      else
        return doSX(double{});
    });
    if (!ok)
      return nullptr;
    o_.release();
    return o;
  }
  if (wantedmethods.m_area && lsf->canCalculateViaArea()) {
    PyObject *o = simpleNew_ownLastDim(ndims - 1, a, (npy_intp)logsiglength,
                                       OutT::typenum);
    if (!o)
      return nullptr;
    auto outp = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
    ReleasableRefHolder o_(o);

    auto doArea = [&](auto dummy) -> bool {
      using InputT = decltype(dummy);
      for (int path = 0; path < nPaths; ++path) {
        char *data = batchPtr(a, 2, path);
        logSigUsingArea<InputT>(data, point_stride, dim_stride, lengthOfPath,
                                lsf->m_level, d, outp + path * logsiglength);
        interrupt();
      }
      return true;
    };
    bool ok = do_interruptible([&] {
      if (use_float32)
        return doArea(float{});
      else
        return doArea(double{});
    });
    if (!ok)
      return nullptr;
    o_.release();
    return o;
  }
  ERR("We had not prepare()d for this request type");
}

static PyObject *logsigbackwards(PyObject *self, PyObject *args) {
  PyObject *a0, *a1, *a2;
  const char *methods = nullptr;
  if (!PyArg_ParseTuple(args, "OOO|z", &a0, &a1, &a2, &methods))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a2);
  if (!lsf)
    return nullptr;

  WantedMethods wantedmethods;
  std::string methodString;
  if (methods)
    methodString = methods;
  if (setWantedMethods(wantedmethods, lsf->m_dim, lsf->m_level, true, true,
                       methodString))
    ERR(wantedmethods.m_errMsg);
  if (!(wantedmethods.m_log_of_signature || wantedmethods.m_expanded ||
        wantedmethods.m_area))
    ERR("Requested method cannot be used here");
  bool needProjectionButDontHaveIt = !wantedmethods.m_expanded &&
                                     !wantedmethods.m_area &&
                                     !lsf->canProjectToBasis();
  // if (needProjectionButDontHaveIt)
  //   ERR("We had not prepared the S method");

  bool is_float32 =
      PyArray_Check(a1) && PyArray_TYPE((PyArrayObject *)a1) == NPY_FLOAT32;
  PyObject *aa = PyArray_FROM_OTF(a1, is_float32 ? NPY_FLOAT32 : NPY_FLOAT64,
                                  NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  bool use_float32 = (PyArray_TYPE(a) == NPY_FLOAT32);

  PyObject *derivs = PyArray_FROM_OTF(a0, NPY_FLOAT64, NPY_ARRAY_ALIGNED);
  if (!derivs)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder derivs_(derivs);
  PyArrayObject *derivsa = (PyArrayObject *)derivs;

  int ndims = PyArray_NDIM(a);
  if (ndims < 2)
    ERR("data must be at least 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, ndims - 2);
  const int d = (int)PyArray_DIM(a, ndims - 1);
  const size_t eachInputLength = (size_t)(lengthOfPath * d);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d != lsf->m_dim)
    ERR(("Path has dimension " + std::to_string(d) +
         " but we prepared for dimension " + std::to_string(lsf->m_dim))
            .c_str());
  if (PyArray_NDIM(derivsa) != ndims - 1)
    ERR("derivs has the wrong number of dimensions");

  int nPaths = 1;
  for (int i = 0; i + 2 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(derivsa, i) != x)
      ERR("mismatched dimensions between sigs and derivs");
    nPaths *= (int)x;
  }

  // Path: use strides directly (zero-copy)
  npy_intp *a_strides = PyArray_STRIDES(a);
  npy_intp point_stride = a_strides[ndims - 2];
  npy_intp dim_stride = a_strides[ndims - 1];

  // Derivs: keep fallback for non-contiguous last dim (rare for 1D-inner)
  ReleasableRefHolder derivs_contig_;
  if (!isInnerContiguous(derivsa, 1)) {
    PyObject *dd2 = PyArray_ContiguousFromAny(a0, NPY_FLOAT64, 0, 0);
    if (!dd2)
      ERR("derivs must be (convertible to) a numpy array");
    derivs_contig_.releaseAndSet(dd2);
    derivsa = (PyArrayObject *)dd2;
  }

  using OutT = UseFloat;
  size_t eachOutputSize = wantedmethods.m_expanded ? (size_t)getSigLength(lsf)
                                                   : lsf->logSigLength();
  if (eachOutputSize != (size_t)PyArray_DIM(derivsa, ndims - 2))
    ERR("derivatives have unexpected length");
  PyObject *o = PyArray_ZEROS(ndims, PyArray_DIMS(a), OutT::typenum, 0);
  if (!o)
    return nullptr;
  auto outp = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  ReleasableRefHolder o_(o);

  Signature sig, sigDer;
  if (needProjectionButDontHaveIt) {
    bool ok = do_interruptible([&] {
      IISignature_algebra::makeSparseLogSigMatrices(lsf->m_dim, lsf->m_level,
                                                    *lsf, interrupt);
      return true;
    });
    ok = ok && postPrepare(lsf);
    if (!ok) {
      // We've failed to add projection info to lsf, we better leave lsf in its
      // safe, original state
      lsf->m_simples.clear();
      lsf->m_smallSVDs.clear();
      lsf->m_smallTriangles.clear();
      return nullptr;
    }
  }
  auto doBackwards = [&](auto dummy) -> bool {
    using InputT = decltype(dummy);
    for (int path = 0; path < nPaths; ++path) {
      const char *data = batchPtr(a, 2, path);
      const double *derivPtr = (const double *)batchPtr(derivsa, 1, path);
      if (wantedmethods.m_area) {
        logSigUsingAreaBackwards<InputT>(
            data, point_stride, dim_stride, lengthOfPath, lsf->m_level, d,
            derivPtr, outp + path * eachInputLength);
      } else {
        CalcSignature::calcSignature<InputT>(
            d, lsf->m_level, lengthOfPath, data, point_stride, dim_stride, sig);
        if (wantedmethods.m_expanded)
          sigDer.fromRaw(lsf->m_dim, lsf->m_level, derivPtr);
        else
          projectExpandedLogSigToBasisBackwards(derivPtr, lsf, sigDer);
        logBackwards(sigDer, sig);
        // This should use the calculated signature
        CalcSignature::sigBackwards<InputT>(
            lsf->m_dim, lsf->m_level, lengthOfPath, data, point_stride,
            dim_stride, sigDer, outp + path * eachInputLength);
      }
    }
    return true;
  };
  bool ok = do_interruptible([&] {
    if (use_float32)
      return doBackwards(float{});
    else
      return doBackwards(double{});
  });
  if (!ok)
    return nullptr;
  o_.release();
  return o;
}

static PyObject *logsigtosig(PyObject *self, PyObject *args) {
  PyObject *a1, *a2;
  if (!PyArg_ParseTuple(args, "OO", &a1, &a2))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a2);
  if (!lsf)
    return nullptr;
  WantedMethods wantedmethods;
  std::string methodString = "S";
  if (setWantedMethods(wantedmethods, lsf->m_dim, lsf->m_level, true, false,
                       methodString))
    ERR(wantedmethods.m_errMsg);
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no logsignatures provided");
  const int logsiglength = (int)PyArray_DIM(a, ndims - 1);
  if (logsiglength != lsf->logSigLength())
    ERR(("data has length " + std::to_string(logsiglength) +
         " but log signatures should have length " +
         std::to_string(lsf->logSigLength()))
            .c_str());
  if (!lsf->m_support_logsig2sig)
    ERR("We did not prepare for log sig -> sig conversion. Ensure '2' is in "
        "the method string in prepare");
  double *data = static_cast<double *>(PyArray_DATA(a));

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    nPaths *= (int)x;
  }

  using OutT = UseDouble;
  PyObject *o = simpleNew_ownLastDim(ndims, a, lsf->m_siglength, OutT::typenum);
  if (!o)
    return nullptr;
  auto outp = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  ReleasableRefHolder o_(o);

  Signature sig;
  bool ok = do_interruptible([&] {
    for (int path = 0; path < nPaths; ++path) {
      logsigToSig(data + path * logsiglength, *lsf,
                  outp + path * lsf->m_siglength);
    }
    return true;
  });
  if (!ok)
    return nullptr;
  o_.release();
  return o;
}

static PyObject *logsigtosigBackwards(PyObject *self, PyObject *args) {
  PyObject *a1, *a2, *a0;
  if (!PyArg_ParseTuple(args, "OOO", &a0, &a1, &a2))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a2);
  if (!lsf)
    return nullptr;
  WantedMethods wantedmethods;
  std::string methodString = "S";
  if (setWantedMethods(wantedmethods, lsf->m_dim, lsf->m_level, true, false,
                       methodString))
    ERR(wantedmethods.m_errMsg);
  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;

  PyObject *derivs = PyArray_ContiguousFromAny(a0, NPY_FLOAT64, 0, 0);
  if (!derivs)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder derivs_(derivs);
  PyArrayObject *derivsa = (PyArrayObject *)derivs;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no logsignatures provided");
  if (ndims != PyArray_NDIM(derivsa))
    ERR("data and derivs must have the same number of dimensions.")
  const int logsiglength = (int)PyArray_DIM(a, ndims - 1);
  if (logsiglength != lsf->logSigLength())
    ERR(("data has length " + std::to_string(logsiglength) +
         " but log signatures should have length " +
         std::to_string(lsf->logSigLength()))
            .c_str());
  if (lsf->m_siglength != (int)PyArray_DIM(derivsa, ndims - 1))
    ERR("derivatives have unexpected length");
  if (!lsf->m_support_logsig2sig)
    ERR("We did not prepare for log sig -> sig conversion. Ensure '2' is in "
        "the method string in prepare");
  double *data = static_cast<double *>(PyArray_DATA(a));
  double *derivsp = static_cast<double *>(PyArray_DATA(derivsa));

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(derivsa, i) != x)
      ERR("mismatched dimensions between data and derivs");
    nPaths *= (int)x;
  }

  using OutT = UseDouble;
  PyObject *o = PyArray_ZEROS(ndims, PyArray_DIMS(a), OutT::typenum, 0);
  if (!o)
    return nullptr;
  auto outp = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  ReleasableRefHolder o_(o);

  Signature sig;
  bool ok = do_interruptible([&] {
    for (int path = 0; path < nPaths; ++path) {
      logsigToSigBackwards(data + path * logsiglength,
                           derivsp + path * lsf->m_siglength, *lsf,
                           outp + path * logsiglength);
    }
    return true;
  });
  if (!ok)
    return nullptr;
  o_.release();
  return o;
}

static PyObject *logsigJoin(PyObject *self, PyObject *args) {
  PyObject *a1, *a2, *a3;
  if (!PyArg_ParseTuple(args, "OOO", &a1, &a2, &a3))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a3);
  if (!lsf)
    return nullptr;

  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("data must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and data must have the same number of dimensions");
  const int d_given = (int)PyArray_DIM(b, ndims - 1);
  if (d_given != lsf->m_dim)
    ERR("step has wrong last dimension");
  const int logSigLength = lsf->logSigLength();
  if ((size_t)logSigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("log signatures have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and data");
    nPaths *= (int)x;
  }

  ReadArrayAsDoubles logsig, displacement;
  if (logsig.read(a, nPaths * logSigLength) ||
      displacement.read(b, nPaths * d_given))
    ERR("Out of memory");

  using OutT = UseDouble;
  PyObject *o = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o)
    return nullptr;
  auto out = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
  auto displacement_ = displacement.ptr();

  FunctionRunner *f = lsf->m_f.get();
  if (f != nullptr || lsf->m_fd.m_length_of_b > 0) {
    const bool useCompiled = f != nullptr;
    for (int i = 0; i < nPaths * logSigLength; ++i)
      out[i] = logsig.ptr()[i];
    for (int path = 0; path < nPaths; ++path) {
      if (useCompiled)
        f->go(out, displacement_);
      else
        slowExplicitFunction(out, displacement_, lsf->m_fd);
      out += logSigLength;
      displacement_ += d_given;
    }
    return o;
  }
  if (lsf->canCalculateViaArea()) {
    for (int path = 0; path < nPaths; ++path) {
      logSigJoinUsingArea(logsig.ptr() + path * logSigLength, displacement_,
                          lsf->m_level, d_given, out);
      out += logSigLength;
      displacement_ += d_given;
    }
    return o;
  }
  ERR("no method")
}

static PyObject *logsigJoinBackwards(PyObject *self, PyObject *args) {
  PyObject *a1, *a2, *a3, *a0;
  if (!PyArg_ParseTuple(args, "OOOO", &a0, &a1, &a2, &a3))
    return nullptr;
  LogSigFunction *lsf = getLogSigFunction(a3);
  if (!lsf)
    return nullptr;

  PyObject *aa = PyArray_ContiguousFromAny(a1, NPY_FLOAT64, 0, 0);
  if (!aa)
    ERR("sigs must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  PyObject *bb = PyArray_ContiguousFromAny(a2, NPY_FLOAT64, 0, 0);
  if (!bb)
    ERR("data must be (convertible to) a numpy array");
  RefHolder b_(bb);
  PyArrayObject *b = (PyArrayObject *)bb;
  PyObject *derivs = PyArray_ContiguousFromAny(a0, NPY_FLOAT64, 0, 0);
  if (!derivs)
    ERR("derivs must be (convertible to) a numpy array");
  RefHolder derivs_(derivs);
  PyArrayObject *derivsa = (PyArrayObject *)derivs;

  int ndims = PyArray_NDIM(a);
  if (ndims < 1)
    ERR("no signatures provided");
  if (PyArray_NDIM(b) != ndims)
    ERR("sigs and data must have the same number of dimensions");
  if (PyArray_NDIM(derivsa) != ndims)
    ERR("sigs and derivs must have the same number of dimensions");
  const int d_given = (int)PyArray_DIM(b, ndims - 1);
  if (d_given != lsf->m_dim)
    ERR("step has wrong last dimension");
  const int logSigLength = lsf->logSigLength();
  if ((size_t)logSigLength != (size_t)PyArray_DIM(a, ndims - 1))
    ERR("log signatures have unexpected length");
  if ((size_t)logSigLength != (size_t)PyArray_DIM(derivsa, ndims - 1))
    ERR("derivs have unexpected length");

  int nPaths = 1;
  for (int i = 0; i + 1 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    if (PyArray_DIM(b, i) != x)
      ERR("mismatched dimensions between sigs and data");
    if (PyArray_DIM(derivsa, i) != x)
      ERR("mismatched dimensions between sigs and derivs");
    nPaths *= (int)x;
  }

  ReadArrayAsDoubles logsig, displacement, deriv;
  if (logsig.read(a, nPaths * logSigLength) ||
      deriv.read(derivsa, nPaths * logSigLength) ||
      displacement.read(b, nPaths * d_given))
    ERR("Out of memory");

  using OutT = UseDouble;
  PyObject *o1 = PyArray_SimpleNew(ndims, PyArray_DIMS(a), OutT::typenum);
  if (!o1)
    return nullptr;
  auto out1 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o1)));
  PyObject *o2 = PyArray_SimpleNew(ndims, PyArray_DIMS(b), OutT::typenum);
  if (!o2)
    return nullptr;
  auto out2 = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o2)));
  auto displacement_ = displacement.ptr();
  auto logsig_p = logsig.ptr();
  auto deriv_p = deriv.ptr();

  if (lsf->m_fd.m_length_of_b > 0) {
    // Using O method
    for (int i = 0; i < nPaths * logSigLength; ++i)
      out1[i] = deriv_p[i];
    for (int i = 0; i < nPaths * d_given; ++i)
      out2[i] = 0;
    for (int path = 0; path < nPaths; ++path) {
      slowExplicitFunctionBackward(logsig_p, displacement_, lsf->m_fd, deriv_p,
                                   out1, out2);
      out1 += logSigLength;
      out2 += d_given;
      deriv_p += logSigLength;
      logsig_p += logSigLength;
      displacement_ += d_given;
    }
    return Py_BuildValue("(NN)", o1, o2);
  }

  if (lsf->canCalculateViaArea()) {
    for (int path = 0; path < nPaths; ++path) {
      logSigJoinUsingAreaBackwards(logsig_p, displacement_, lsf->m_level,
                                   d_given, deriv_p, out1, out2);
      out1 += logSigLength;
      out2 += d_given;
      deriv_p += logSigLength;
      logsig_p += logSigLength;
      displacement_ += d_given;
    }
    return Py_BuildValue("(NN)", o1, o2);
  }
  ERR("no method available: currently need level <= 2")
}

#endif

const char *const rotInv_id = "iisignature.rotInv";
RotationalInvariants::Prepared *getPreparedRotInv(PyObject *p) {
  if (!PyCapsule_IsValid(p, rotInv_id))
    ERR("I have received an input which is not from "
        "iisignature.rotinv2dprepare()");
  return (RotationalInvariants::Prepared *)PyCapsule_GetPointer(p, rotInv_id);
}

static void killRotInv(PyObject *p) { delete getPreparedRotInv(p); }

static PyObject *rotinv2dlength(PyObject *self, PyObject *args) {
  /*
    int level = 0;
    if (!PyArg_ParseTuple(args, "i", &level))
      return nullptr;
    if (level < 2 || level > 63 || level % 2 != 0)
      ERR("Level must be a small positive even number");
    long ans = 0;
    for (int lev = 2; lev <= level; lev += 2)
      //length is (lev choose lev/2)
      ans += (long)LogSigLength::centralBinomialCoefficient(lev);*/
  PyObject *a1;
  if (!PyArg_ParseTuple(args, "O", &a1))
    return nullptr;
  auto prepared = getPreparedRotInv(a1);
  if (!prepared)
    return nullptr;
  long ans = 0;
  for (const auto &i : prepared->m_invariants)
    ans += (long)i.size();

  return PyLong_FromLong(ans);
}

static PyObject *rotinv2dprepare(PyObject *self, PyObject *args) {
  int level = 0;
  const char *type = nullptr;
  if (!PyArg_ParseTuple(args, "iz", &level, &type))
    return nullptr;
  if (level < 2 || level > 63 || level % 2 != 0)
    ERR("Level must be a small positive even number");
  RotationalInvariants::InvariantType t;
  if (type) {
    if (RotationalInvariants::getWantedMethod(type, t))
      ERR("Invalid type of rotational invariant")
  } else
    t = RotationalInvariants::InvariantType::SVD;

  using T = RotationalInvariants::Prepared;
  auto p = std::unique_ptr<T>(new T(level, t));
#ifndef IISIGNATURE_NO_NUMPY
  if (t == RotationalInvariants::InvariantType::SVD ||
      t == RotationalInvariants::InvariantType::QR) {
    LeastSquares ls(t == RotationalInvariants::InvariantType::QR);
    if (!ls.m_ok)
      return nullptr;
    vector<double> tempa, tempb, tempc;
    for (int lev = 4; lev <= level; lev += 2) {
      for (int parity : {0, 1}) {
        size_t idx = lev - 2 + parity;
        RotationalInvariants::invariantsToMatrix(p->m_invariants[idx], lev,
                                                 tempa);
        RotationalInvariants::invariantsToMatrix(p->m_knownInvariants[idx], lev,
                                                 tempb);
        bool ok = false;
        if (t == RotationalInvariants::InvariantType::QR)
          ok = ls.projectAwayFromQR(tempa, tempb, ((npy_intp)1) << (lev - 1),
                                    tempc);
        else
          ok = ls.projectAwayFromSVD(tempa, tempb, ((npy_intp)1) << (lev - 1),
                                     tempc);
        if (!ok)
          return nullptr;
        RotationalInvariants::invariantsFromMatrix(tempc, lev, parity,
                                                   p->m_invariants[idx]);
      }
    }
  }
#endif
  if (t == RotationalInvariants::InvariantType::KNOWN)
    p->m_invariants = p->m_knownInvariants;
  PyObject *out = PyCapsule_New(p.release(), rotInv_id, killRotInv);
  return out;
}

#ifndef IISIGNATURE_NO_NUMPY
// iisignature.rotinv2d(numpy.random.uniform(size=(12,2)),4)
static PyObject *rotinv2d(PyObject *self, PyObject *args) {
  PyObject *a1, *a2;
  if (!PyArg_ParseTuple(args, "OO", &a1, &a2))
    return nullptr;
  auto prepared = getPreparedRotInv(a2);
  if (!prepared)
    return nullptr;
  int level = prepared->m_level;
  size_t nInvariants = 0;
  for (const auto &i : prepared->m_invariants)
    nInvariants += (long)i.size();

  bool is_float32 =
      PyArray_Check(a1) && PyArray_TYPE((PyArrayObject *)a1) == NPY_FLOAT32;
  PyObject *aa = PyArray_FROM_OTF(a1, is_float32 ? NPY_FLOAT32 : NPY_FLOAT64,
                                  NPY_ARRAY_ALIGNED);
  if (!aa)
    ERR("data must be (convertible to) a numpy array");
  RefHolder a_(aa);
  PyArrayObject *a = (PyArrayObject *)aa;
  bool use_float32 = (PyArray_TYPE(a) == NPY_FLOAT32);
  int ndims = PyArray_NDIM(a);
  if (ndims < 2)
    ERR("data must be 2d");
  const int lengthOfPath = (int)PyArray_DIM(a, ndims - 2);
  const int d = (int)PyArray_DIM(a, ndims - 1);
  if (lengthOfPath < 1)
    ERR("Path has no length");
  if (d != 2)
    ERR(("Path has dimension " + std::to_string(d) +
         " but must have dimension 2.")
            .c_str());
  if (level < 2 || level > 63 || level % 2 != 0)
    ERR("Level must be a small positive even number");
  int nPaths = 1;
  for (int i = 0; i + 2 < ndims; ++i) {
    npy_intp x = PyArray_DIM(a, i);
    nPaths *= (int)x;
  }
  npy_intp *strides = PyArray_STRIDES(a);
  npy_intp point_stride = strides[ndims - 2];
  npy_intp dim_stride = strides[ndims - 1];
  using OutT = UseDouble;
  PyObject *o = simpleNew_ownLastDim(ndims - 1, a, nInvariants, OutT::typenum);
  if (!o)
    return nullptr;
  ReleasableRefHolder o_(o);
  OutT::T *out_data = static_cast<OutT::T *>(
      PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));

  Signature sig;
  std::vector<double> out(nInvariants);
  auto doRotInv = [&](auto dummy) -> bool {
    using InputT = decltype(dummy);
    for (int path = 0; path < nPaths; ++path) {
      char *in_data = batchPtr(a, 2, path);
      if (!calcSignature<InputT>(sig, in_data, point_stride, dim_stride,
                                 lengthOfPath, d, level))
        return false;
      for (int lev = 2; lev <= level; lev += 2) {
        for (int parity : {0, 1}) {
          size_t idx = lev - 2 + parity;
          auto &invs = prepared->m_invariants[idx];
          for (auto &i : invs) {
            double d = 0;
            for (auto &p : i) {
              d += p.second * sig.m_data[lev - 1][(size_t)(p.first)];
            }
            *(out_data++) = d;
          }
        }
      }
    }
    return true;
  };
  bool ok = do_interruptible([&] {
    if (use_float32)
      return doRotInv(float{});
    else
      return doRotInv(double{});
  });
  if (!ok)
    return nullptr;

  o_.release();
  return o;
}

static PyObject *rotinv2dcoeffs(PyObject *self, PyObject *args) {
  PyObject *a1;
  if (!PyArg_ParseTuple(args, "O", &a1))
    return nullptr;
  auto prepared = getPreparedRotInv(a1);
  if (!prepared)
    return nullptr;
  int level = prepared->m_level;

  PyObject *out = PyTuple_New(level / 2);
  using OutT = UseDouble;
  for (int lev = 2; lev <= level; lev += 2) {
    auto &source0 = prepared->m_invariants[lev - 2];
    auto &source1 = prepared->m_invariants[lev - 1];
    size_t sourceSize = source0.size() + source1.size();
    size_t length = ((size_t)1u) << lev;
    npy_intp dims[] = {(npy_intp)(sourceSize), (npy_intp)length};
    PyObject *o = PyArray_SimpleNew(2, dims, OutT::typenum);
    if (!o)
      return nullptr; // but leak out...
    OutT::T *dest = static_cast<OutT::T *>(
        PyArray_DATA(reinterpret_cast<PyArrayObject *>(o)));
    for (size_t i = 0; i < length * sourceSize; ++i)
      dest[i] = 0;
    for (size_t i = 0; i < source0.size(); ++i)
      for (const auto &p : source0[i])
        dest[i * length + p.first] = p.second;
    for (size_t i = 0; i < source1.size(); ++i)
      for (const auto &p : source1[i])
        dest[(i + source0.size()) * length + p.first] = p.second;
    PyTuple_SET_ITEM(out, lev / 2 - 1, o);
  }

  return out;
}
#endif // IISIGNATURE_NO_NUMPY

#define METHOD_DESC                                                            \
  "some combination of 'D' (the default), "                                    \
  "'C' (the bch formula compiled on the fly), "                                \
  "'O' (the bch formula evaluated simply and stored in an object without "     \
  "on-the-fly compilation and perhaps more slowly), "                          \
  "'A' (simple area calculation, useful for level <=2), "                      \
  "'S' (calculating the log signature by first calculating "                   \
  "the signature and then taking its log, "                                    \
  "which may be faster for high levels or long paths), "                       \
  "and 'X' (which is a bit special: means the log signature returned "         \
  "expanded in tensor space)"

static PyMethodDef Methods[] = {
#ifndef IISIGNATURE_NO_NUMPY
    {"sig", sig, METH_VARARGS,
     "sig(X,m,format=0)\n Returns the signature of a path X "
     "up to level m as an array of shape (...,siglength(D,m)). X must be "
     "convertible to a numpy [...x]NxD float32 or float64 array of points "
     "making up the path in R^D. The initial 1 in the zeroth level of the "
     "signature is excluded. "
     "If format is 1, the output is a tuple of arrays, one for each level, not "
     "a single one. "
     "If format is 2, the output is an array of shape [...,N-1,siglength(D,m)] "
     "of all the cumulative signatures"
     " from the first point to each other point."},
    {"sigmultcount", sigMultCount, METH_VARARGS,
     "sigmultcount(X,m)\n "
     "Returns the number of multiplications which sig(X,m) would perform."},
    {"sigjacobian", sigJacobian, METH_VARARGS,
     "sigjacobian(X,m)\n "
     "Returns the full Jacobian matrix of "
     "derivatives of sig(X,m) with respect to X. "
     "If X is an NxD array then the output is an NxDx(siglength(D,m)) array."},
    {"sigbackprop", sigBackwards, METH_VARARGS,
     "sigbackprop(s,X,m)\n "
     "If s is the derivative of something with respect to sig(X,m), "
     "then this returns the derivative of that thing with respect to X. "
     "sigbackprop(s,X,m) should be approximately "
     "numpy.dot(sigjacobian(X,m),s)"},
    {"sigjoin", sigJoin, METH_VARARGS,
     "sigjoin(X,D,m,f=float('nan'))\n "
     "If X is an array of signatures of d dimensional paths of shape "
     "(..., siglength(d,m)) and D is an array of d dimensional displacements "
     "of shape (..., d), then return an array shaped like X "
     "of the signatures of the paths concatenated with the displacements. "
     "If f is provided, then it is taken to be the fixed value of the "
     "displacement in the last dimension, and D should have shape (K, d-1)."},
    {"sigjoinbackprop", sigJoinBackwards, METH_VARARGS,
     "sigjoinbackprop(s,X,D,m,f=float('nan')) \n "
     "gives the derivatives of F with respect to X and D (and f if given) "
     "where s is the derivatives"
     " of F with respect to sigjoin(X,D,m,f). The result is a tuple of two or "
     "three items."},
    {"sigcombine", sigCombine, METH_VARARGS,
     "sigcombine(X1,X2,d,m)\n "
     "If X1 and X2 are arrays of signatures of d dimensional paths of shape "
     "(..., siglength(d,m)), then return an array of the same shape "
     "of the signatures of each path from X1 concatenated with each path from "
     "X2. "
     "In other words, this is the concatenation/Chen product of two "
     "signatures."},
    {"sigcombinebackprop", sigCombineBackwards, METH_VARARGS,
     "sigcombinebackprop(s,X1,X2,d,m) \n "
     "gives the derivatives of F with respect to X1 and X2 where s is the "
     "derivatives"
     " of F with respect to sigcombine(X1,X2,d,m). The result is a tuple of "
     "two items."},
    {"sigscale", sigScale, METH_VARARGS,
     "sigscale(X,D,m)\n "
     "If X is an array of signatures of d dimensional paths of shape "
     "(..., siglength(d,m)) and D is an array of d dimensional scales "
     "of shape (..., d), then return an array shaped like X "
     "of the signatures of the paths scaled by the corresponding scale factor "
     "in each dimension. "},
    {"sigscalebackprop", sigScaleBackwards, METH_VARARGS,
     "sigscalebackprop(s,X,D,m) \n "
     "gives the derivatives of F with respect to X and D where s is the "
     "derivatives"
     " of F with respect to sigscale(X,D,m). The result is a tuple of two "
     "items."},
#endif
    {"siglength", siglength, METH_VARARGS,
     "siglength(d,m) \n "
     "Returns the length of the signature (excluding the initial 1) of a d "
     "dimensional path up to level m"},
    {"logsiglength", logsiglength, METH_VARARGS,
     "logsiglength(d,m) \n "
     "Returns the length of the log signature of a d dimensional path up to "
     "level m"},
    {"rotinv2dlength", rotinv2dlength, METH_VARARGS,
     "rotinv2dlength(s) \n "
     "Returns the number of linear rotational invariants which rotinv2d(X,s) "
     "will return. "
     "s must be the result of a call to rotinv2dprepare."},
    {"rotinv2dprepare", rotinv2dprepare, METH_VARARGS,
     "rotinv2dprepare(m, type) \n "
     "This prepares the way to find linear rotational invariants of signatures "
     "up to level m "
     "of 2d paths. m must be even. The returned object is used in rotinv2d. "
     "type should be "
     "'a' to return all invariants, or 's' to use SVD to exclude those which "
     "are determined by "
     "shuffle products of lower level invariants."},
    {"prepare", prepare, METH_VARARGS,
     "prepare(d, m, methods=None) \n "
     "This prepares the way to calculate log signatures of d dimensional paths"
     " up to level m. The returned object is used in the logsig, basis and "
     "info functions. \n"
     " By default, only the default method will be prepared, but you can "
     "change it "
     "by setting methods to " METHOD_DESC
     ". Adding 'H' to methods makes the basis used be "
     "the standard Hall basis, otherwise the Lyndon basis is used. Adding '2' "
     "allows the "
     "logsigtosig function to be used."},
    {"basis", basis, METH_VARARGS,
     "basis(s) \n  Returns a tuple of strings "
     "which are the basis elements of the log signature. s must have come from "
     "prepare."
     " This function is work in progress, especially for dimension greater "
     "than 8. "
     "An example of how to parse the output of this function can be seen in "
     "the tests."},
    {"info", info, METH_VARARGS,
     "info(s) \n  Returns a dictionary of "
     "information about the opaque object s. s must have come from prepare."},
#ifndef IISIGNATURE_NO_NUMPY
    {"logsig", logsig, METH_VARARGS,
     "logsig(X, s, methods=None) \n "
     "Calculates the log signature of the path X. X must be a numpy [...x]Nxd "
     "float32 "
     "or float64 array of points making up the path in R^d. s must have come "
     "from "
     "prepare(d,m) for some m. The value is returned as a 1D numpy array of "
     "its "
     "log signature up to level m. By default, the method used is the default "
     "out "
     "of those which have been prepared, "
     "but you can restrict it by setting methods to " METHOD_DESC "."},
    {"logsigbackprop", logsigbackwards, METH_VARARGS,
     "logsigbackprop(y, X, s, methods=None) \n"
     "gives the derivatives of F with respect to X where y is the derivatives"
     " of F with respect to logsig(X,s,methods). The result has the same shape "
     "as X."},
    {"logsigtosig", logsigtosig, METH_VARARGS,
     "logsigtosig(X, s) \n "
     "Returns the signature of the path whose log signature is X. X must be a "
     "numpy [...xL] float32 "
     "or float64 array of log(signatures) making up the path in R^d. s must "
     "have come from "
     "prepare(d,m,method) for some m, where L is logsiglength(d,m) and the "
     "method string contains "
     "the number 2. The value is returned as a numpy array of shape "
     "[...,iisignature.siglength(d,m)]."},
    {"logsigtosigbackprop", logsigtosigBackwards, METH_VARARGS,
     "logsigtosigbackprop(y, X, s) \n "
     "gives the derivatives of F with respect to X where y is the derivatives"
     " of F with respect to logsigtosig(X,s). The result has the same shape as "
     "X."},
    {"logsigjoin", logsigJoin, METH_VARARGS,
     "logsigjoin(X,D,s)\n "
     "If X is an array of log signatures of d dimensional paths of shape "
     "(..., logsiglength(d,m)) and D is an array of d dimensional "
     "displacements "
     "of shape (..., d), then return an array shaped like X "
     "of the logsignatures of the paths concatenated with the displacements. "
     "s must have come from prepare(d,m) for some m and include methods C O or "
     "A."},
    {"logsigjoinbackprop", logsigJoinBackwards, METH_VARARGS,
     "logsigjoinbackprop(y,X,D,s)\n "
     "gives the derivatives of F with respect to X and D where y is the "
     "derivatives"
     " of F with respect to logsigjoin(X,D,s). The result is a tuple of two "
     "items. "
     "This is only implemented for the A and O methods."},
    {"rotinv2d", rotinv2d, METH_VARARGS,
     "rotinv2d(X,s) \n "
     "Calculates the linear rotational invariants of the signature of the path "
     "X. X must be a "
     "[...x]Nx2 array of points making up the path(s) in R^2. "
     "The value is returned as a 1D numpy array "
     "of invariants from the signature. s must be the result of a call to "
     "rotinv2dprepare."},
    {"rotinv2dcoeffs", rotinv2dcoeffs, METH_VARARGS,
     "rotinv2dcoeffs(s) \n "
     "Returns the basis of rotational invariants as a tuple of 2D numpy arrays "
     "- (nInvariants x length) for each even level. "
     "s must be the result of a call to rotinv2dprepare."},
#endif
    {"version", version, METH_NOARGS, "return the iisignature version string"},
    {NULL, NULL, 0, NULL} /* Sentinel */
};

#define MODULEDOC                                                              \
  "iisignature: Iterated integral signature and logsignature calculations"     \
  "\n\nPlease find documentation at http://www2.warwick.ac.uk/jreizenstein"    \
  " and code at https://github.com/bottler/iisignature."

static struct PyModuleDef moduledef = {
    PyModuleDef_HEAD_INIT,
    "iisignature", /* m_name */
    MODULEDOC,     /* m_doc */
    -1,            /* m_size */
    Methods,       /* m_methods */
    NULL,          /* m_reload */
    NULL,          /* m_traverse */
    NULL,          /* m_clear */
    NULL,          /* m_free */
};

PyMODINIT_FUNC PyInit_iisignature(void) {
#ifndef IISIGNATURE_NO_NUMPY
  import_array();
#endif
  PyObject *moduleObj = PyModule_Create(&moduledef);
  PyModule_AddStringConstant(moduleObj, "__version__", TOSTRING(VERSION));
  return moduleObj;
}

/*
thinking about builds:
According to
http://python-packaging-user-guide.readthedocs.io/en/latest/distributing/#requirements-for-packaging-and-distributing
we can't distribute platform wheels on linux.
Basically we can only distribute source.
 Therefore we won't make wheels

 Build just with
python setup.py sdist

*/
