//////////////////////////////////////////////////////////////////////////
// SymmetricSubsetGraphMasterMaster.hh
// produced: 29/03/2020 jr
/////////////////////////////////////////////////////////////////////////
#ifndef SYMMETRICSUBSETGRAPHMASTER_HH
#define SYMMETRICSUBSETGRAPHMASTER_HH

#include <iostream>
#include <iomanip>
#include <fstream>
#include <string>
#include <sstream>
#include <filesystem>

#include <assert.h>
#include <deque>
#include <vector>
#include <set>
#include <map>

#include <thread>
#include <mutex>
// #include <shared_mutex>
#include <atomic>
#include <condition_variable>

#include "Signal.hh"
#include "Global.hh"
#include "CommandlineOptions.hh"
#include "Statistics.hh"
#include "Graphics.hh"
#include "StrictStairCaseMatrixTrans.hh"
#include "LabelSet.hh"
#include "PointConfiguration.hh"
#include "Chirotope.hh"
#include "Symmetry.hh"
#include "ClassifiedSubsetSymmetries.hh"
#include "SymmetricSubsetGraph.hh"
#include "ScreenReport.hh"
#include "CheckPoint.hh"
#include "Message.hh"

namespace topcom {

  template <ssg_mode_type mode>
  class SymmetricSubsetGraphMaster;

  template <ssg_mode_type mode>
  class SymmetricSubsetGraphMaster {
  public:
    typedef typename SymmetricSubsetGraph<mode>::node_type node_type;
  private:
    class Worker {
    private:

      // context information:
      const int                            _workerID;
      const SymmetricSubsetGraphMaster*    _callerptr;
      const node_type*                     _root_nodeptr;

      // results:
      size_type                            _totalcount;
      size_type                            _symcount;
      size_type                            _nodecount;
      size_type                            _deadendcount;
      size_type                            _earlydeadendcount;
      std::deque<node_type>                _open_nodes;

      // statistics:
      size_type                            _no_of_runs;

      // multi-threading:
      enum State { idle, hired, busy, done, interrupted, stopped };
      State                                _state;
      bool                                 _worker_stopped;
    public:
      mutable std::condition_variable      worker_condition;
    private:
      Worker();
    public:
      Worker(const int,
	     const SymmetricSubsetGraphMaster&);
      Worker(const Worker&);
    private:
      const Worker& operator=(const Worker&);
    public:

      // accessors:
      inline const int                    workerID()          const { return _workerID; }
      inline const size_type              totalcount()        const { return _totalcount; }
      inline const size_type              symcount()          const { return _symcount; }
      inline const size_type              nodecount()         const { return _nodecount; }
      inline const size_type              deadendcount()      const { return _deadendcount; }
      inline const size_type              earlydeadendcount() const { return _earlydeadendcount; }

      // accessor to the no of runs:
      inline const size_type              no_of_runs()        const { return _no_of_runs; }
      inline const std::deque<node_type>& open_nodes()        const { return _open_nodes; }

      // accessors for multi-threading control:
      inline const bool                   check_state(const State state) const {
	std::lock_guard<std::mutex> main_lock(_callerptr->_main_waitmutex);
	return (_state == state);
      }
      inline const bool                   is_idle()           const { return (check_state(State::idle)); }
      inline const bool                   is_hired()          const { return (check_state(State::hired)); }
      inline const bool                   is_busy()           const { return (check_state(State::busy)); }
      inline const bool                   is_done()           const { return (check_state(State::done)); }
      inline const bool                   is_interrupted()    const { return (check_state(State::interrupted)); }
      inline const bool                   is_stopped()        const { return (check_state(State::stopped)); }

      // modifiers for multi-threading control:
      inline void                         set_state(const State state) {
	std::lock_guard<std::mutex> main_lock(_callerptr->_main_waitmutex);
	_state = state;
      }
      inline void                         set_idle()                { set_state(State::idle); }
      inline void                         set_hired()               { set_state(State::hired); }
      inline void                         set_busy()                { set_state(State::busy); }
      inline void                         set_done()                { set_state(State::done); }
      inline void                         set_interrupted()         { set_state(State::interrupted); }
      inline void                         set_stopped()             { set_state(State::stopped); }
      inline const bool                   wake_up()           const;
      inline void                         interrupt_worker();
      inline void                         stop_worker();
      inline void                         pass_work(const node_type*);
      inline void                         clear_results();
      inline void                         run();
      void                                operator()();
    };
  private:
    static const std::string             _dump_startline;
    static const std::string             _dump_endline;
  private:
    const parameter_type                 _no;
    const parameter_type                 _rank;
    const PointConfiguration*            _pointsptr;
    const Chirotope*                     _chiroptr; // currently unused
    const SymmetryGroup*                 _symmetriesptr;
    const ClassifiedSubsetSymmetries*    _classified_symmetriesptr;
    const SwitchTable<LabelSet,
		      colexmax_mode>*    _switch_tableptr;
    node_type*                           _root_nodeptr;
    bool                                 _print_objects;
    bool                                 _save_objects;
  private:
    size_type                            _node_budget;
  private:
    size_type                            _totalcount;
    size_type                            _symcount;
    size_type                            _nodecount;
    size_type                            _deadendcount;
    size_type                            _earlydeadendcount;
    std::deque<node_type>                _open_nodes;
    size_type                            _current_workbuffersize;
    bool                                 _interrupt;

    // this container is for cocircuit enumeration via independent bases:
    mutable ssg_doneset_type             _result_set;
  private:

    // checkpointing:
    ScreenReport                         _screenreport;
    CheckPoint                           _checkpoint;
    // long                                 _no_report_count;
    // long                                 _no_checkpoint_count;
    // int                                  _dump_no;
    // std::fstream                         _dump_str;
  private:

    // multi-threading:
    int                                  _no_of_threads;
    std::vector<std::thread>             _threads;
    std::vector<Worker>                  _workers;
    mutable int                          _no_of_idle_threads;
    mutable int                          _no_of_pending_results;
    // mutable std::shared_mutex            _main_datamutex;
    mutable std::mutex                   _main_waitmutex;
    mutable std::condition_variable      _main_condition;
    friend class SymmetricSubsetGraphMaster::Worker;

  private:
    SymmetricSubsetGraphMaster();
    SymmetricSubsetGraphMaster(const SymmetricSubsetGraphMaster&);
    SymmetricSubsetGraphMaster& operator=(const SymmetricSubsetGraphMaster&);
  public:

    // constructor:
    inline SymmetricSubsetGraphMaster(const parameter_type, 
				      const parameter_type,
				      const PointConfiguration&,
				      const SymmetryGroup&,
				      const bool,
				      const bool);

    // destructor:
    inline ~SymmetricSubsetGraphMaster();

    // accessors:
    inline const int get_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      return _no_of_idle_threads;
    }
    inline const int get_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      return _no_of_pending_results;
    }

    // modifiers:
    inline void set_no_of_idle_threads(const int i) const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      _no_of_idle_threads = i;
    }
    inline void set_no_of_pending_results(const int i) const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      _no_of_pending_results = i;
    }
    inline void increase_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      ++_no_of_idle_threads;
    }
    inline void increase_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      ++_no_of_pending_results;
    }
    inline void decrease_no_of_idle_threads() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      --_no_of_idle_threads;
    }
    inline void decrease_no_of_pending_results() const {
      std::lock_guard<std::mutex> main_lock(_main_waitmutex);
      --_no_of_pending_results;
    }

    // adapting to workload:
    inline void update_node_budget();

    // reporting:
    inline Message& report_progress(Message&, bool force = false);

    // checkpointing:
    inline void trigger_checkpoint(bool force = false);
    inline void interrupt_workers();
    inline void uninterrupt_workers();
    inline void report_checkpoint();

    // results:
    inline const size_type totalcount()               const { return _totalcount; }
    inline const size_type symcount()                 const { return _symcount; }
    inline const size_type nodecount()                const { return _nodecount; }
    inline const size_type deadendcount()             const { return _deadendcount; }
    inline const size_type earlydeadendcount()        const { return _earlydeadendcount; }
    inline const ssg_doneset_type resultset()         const { return _result_set; }

    // multi-threading control:
    inline const bool wake_up() const;

    // internal methods:
    inline const bool _exists_idle_thread() const;
    inline const bool _exists_pending_result() const;
    inline const bool _all_threads_idle() const;
    inline void _collect_results();
    void _run();
    void _init();
    void _init_workers();

    // stream output/input:
    std::ostream& write(std::ostream&) const;
    inline friend std::ostream& operator<<(std::ostream& ost, const SymmetricSubsetGraphMaster& ssgm) {
      return ssgm.write(ost);
    }
    std::istream& read(std::istream&);
    inline friend std::istream& operator>>(std::istream& ist, SymmetricSubsetGraphMaster& ssgm) {
      return ssgm.read(ist);
    }
  };

  template <ssg_mode_type mode>
  const std::string SymmetricSubsetGraphMaster<mode>::_dump_startline = "TOPCOM SymmetricSubsetGraphMaster dump start:";
  template <ssg_mode_type mode>
  const std::string SymmetricSubsetGraphMaster<mode>::_dump_endline   = "SymmetricSubsetGraphMaster dump end.";

  // inlines for worker subclass:
  template <ssg_mode_type mode>
  SymmetricSubsetGraphMaster<mode>::Worker::Worker(const int                               workerID,
						   const SymmetricSubsetGraphMaster<mode>& ssgm) :
    _workerID(workerID),
    _callerptr(&ssgm),
    _root_nodeptr(0),
    _symcount(0UL),
    _totalcount(0UL),
    _nodecount(0UL),
    _deadendcount(0UL),
    _earlydeadendcount(0UL),
    _no_of_runs(0),
    _open_nodes(),
    _state(idle),
    _worker_stopped(false) {
  }

  template <ssg_mode_type mode>
  SymmetricSubsetGraphMaster<mode>::Worker::Worker(const Worker& sw) :
    _workerID(sw._workerID),
    _callerptr(sw._callerptr),
    _root_nodeptr(sw._root_nodeptr),
    _symcount(sw._symcount),
    _totalcount(sw._totalcount),
    _nodecount(sw._nodecount),
    _deadendcount(sw._deadendcount),
    _earlydeadendcount(sw._earlydeadendcount),
    _no_of_runs(sw._no_of_runs),
    _open_nodes(sw._open_nodes),
    _state(idle),
    _worker_stopped(false) {
  }


  // shared data: _worker_stopped (read)
  // callers: all worker threads
  template <ssg_mode_type mode>
  inline const bool SymmetricSubsetGraphMaster<mode>::Worker::wake_up() const {

    // since this function is called inside the condition variable predicate,
    // we need to access the shared data lock-free (the lock is already held by the wait):
    return ((_state == State::hired) || (_state == State::stopped));
  }

  // shared data: _state, _callerptr->_no_of_idle_threads (write)
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::Worker::pass_work(const SymmetricSubsetGraphMaster<mode>::node_type* nodeptr) {
    _callerptr->decrease_no_of_idle_threads();
    MessageStreams::debug() << message::lock
			    << "worker " << _workerID << " is assigned to work on "
			    << *nodeptr << " ..." << std::endl
			    << message::unlock;
    _root_nodeptr = nodeptr;
    set_hired();
    MessageStreams::debug() << message::lock
			    << "... done" << std::endl
			    << message::unlock;
  }

  // shared data: _state, _callerptr->_no_of_pending_results (write)
  // shared const data: most of _callerptr->...
  // callers: all worker threads
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::Worker::run() {
    set_busy();
    SymmetricSubsetGraph<mode> ssg(_workerID,
				   _no_of_runs,
				   _callerptr->_no,
				   _callerptr->_rank,
				   _callerptr->_pointsptr,
				   _callerptr->_symmetriesptr,
				   _callerptr->_classified_symmetriesptr,
				   _callerptr->_switch_tableptr,
				   _root_nodeptr,
				   _open_nodes,
				   _callerptr->_result_set,
				   _callerptr->_print_objects,
				   _callerptr->_save_objects,
				   _callerptr->_node_budget,
				   &_callerptr->_current_workbuffersize,
				   &_callerptr->_interrupt);
    _nodecount                = ssg.nodecount();
    _totalcount               = ssg.totalcount();
    _symcount                 = ssg.symcount();
    _deadendcount             = ssg.deadendcount();
    _earlydeadendcount        = ssg.earlydeadendcount();
#ifdef COMPUTATIONS_DEBUG
    MessageStreams::debug() << message::lock
			    << "... copying remaning work into worker open queue ..." << std::endl
			    << message::unlock;
#endif
#ifdef COMPUTATIONS_DEBUG
    MessageStreams::debug() << message::lock
			    << "... done" << std::endl
			    << message::unlock;
#endif
    ++_no_of_runs;
    set_done();
    _callerptr->increase_no_of_pending_results();
  }

  // shared data: _state, _callerptr->_no_of_pending_results, _callerptr->_no_of_idle_threads (write)
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::Worker::clear_results() {
    if (_root_nodeptr) {
      delete(_root_nodeptr);
    }
    _root_nodeptr            = 0;
    _symcount                = 0UL;
    _totalcount              = 0UL;
    _nodecount               = 0UL;
    _deadendcount            = 0UL;
    _earlydeadendcount       = 0UL;
    _open_nodes.clear();
    _callerptr->decrease_no_of_pending_results();
    set_idle();
    _callerptr->increase_no_of_idle_threads();
  }
  
  // shared data: _worker_stopped (write)
  // callers: main thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::Worker::stop_worker() {
    set_stopped();
  }

  // shared data: _worker_stopped (read)
  // callers: all worker threads
  template <ssg_mode_type mode>
  void SymmetricSubsetGraphMaster<mode>::Worker::operator()() {
    
    // main loop of worker thread:
    MessageStreams::debug() << message::lock
			    << message::tab << "worker " << _workerID << " is spawned" << std::endl
			    << message::unlock;

    // do not notify many times in a row:
    bool master_notified = false;
    while (!is_stopped()) {
     
      // signal handling:
      if (Signal::signal_received()) {

	// give the master the opportunity to handle the signal:
	_callerptr->_main_condition.notify_one();
      }
 
      // the actual work is done here:
      if (is_hired()) {
	MessageStreams::debug() << message::lock
				<< message::tab << "worker " << _workerID << " doing work ..." << std::endl
				<< message::unlock;
	run();
       	master_notified = false;
	MessageStreams::debug() << message::lock
				<< message::tab << "... worker " << _workerID << " done" << std::endl
				<< message::unlock;
      }
      else if (is_done() && !master_notified) {
	
	// actual work done:
	MessageStreams::debug() << message::lock
				<< message::tab << "... worker " << _workerID << " done - notifying master" << std::endl
				<< message::unlock;
	_callerptr->_main_condition.notify_one();
	master_notified = true;
      }
      
      // no work - wait:
      else if (is_idle()) {
	MessageStreams::debug() << message::lock
				<< message::tab << "worker " << _workerID << " idle - wait" << std::endl
				<< message::unlock;
	while (!wake_up()) {
	  std::unique_lock<std::mutex> main_lock(_callerptr->_main_waitmutex);
	  worker_condition.wait(main_lock, [this] { return wake_up(); });
	}
	MessageStreams::debug() << message::lock
				<< message::tab << "worker " << _workerID << " waking up ..." << std::endl
				<< message::unlock;
      }
    }
    MessageStreams::debug() << message::lock
			    << message::tab << "worker " << _workerID << " terminating" << std::endl
			    << message::unlock;
  }

  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::Worker::interrupt_worker() {
    set_interrupted();
  }

  // inlines for the main class:
  template <ssg_mode_type mode>
  inline SymmetricSubsetGraphMaster<mode>::SymmetricSubsetGraphMaster(const parameter_type no, 
								      const parameter_type rank,
								      const PointConfiguration& points,
								      const SymmetryGroup& symmetries,
								      const bool print_objects,
								      const bool save_objects) :
    _no(no),
    _rank(rank),
    _pointsptr(&points),
    _chiroptr(0),  // currently unused
    _symmetriesptr(&symmetries),
    _classified_symmetriesptr(0),
    _switch_tableptr(0),
    _root_nodeptr(0),
    _print_objects(print_objects),
    _save_objects(save_objects),
    _node_budget(1),
    _totalcount(0),
    _symcount(0),
    _nodecount(0),
    _deadendcount(0),
    _earlydeadendcount(0),
    _open_nodes(),
    _current_workbuffersize(0UL),
    _interrupt(false),
    _result_set(),
    _screenreport(),
    _checkpoint(),
    _no_of_threads(1) {
    _init();
    
    // initialize signal handling:
    Signal::init();

    _run();
  }

  template <ssg_mode_type mode>
  inline SymmetricSubsetGraphMaster<mode>::~SymmetricSubsetGraphMaster() {
#ifdef CONSTRUCTOR_DEBUG
    MessageStreams::debug() << message::lock
			    << "SymmetricSubsetGraphMaster<mode>::~SymmetricSubsetGraphMaster():" << std::endl
			    << "destructing owned pointers ..." << std::endl
			    << message::unlock;
#endif
    if (CommandlineOptions::parallel_enumeration()) {
      MessageStreams::debug() << message::lock
			      << "work completed - master is stopping all workers ..." << std::endl
			      << message::unlock;
      for (int i = 0; i < _no_of_threads; ++i) {
	_workers[i].stop_worker();
      }
      MessageStreams::debug() << message::lock
			      << "master is waking up all workers for termination ..." << std::endl
			      << message::unlock;
      for (int i = 0; i < _no_of_threads; ++i) {
	_workers[i].worker_condition.notify_one();
      }
      MessageStreams::debug() << message::lock
			      << "master is waiting for threads to join ..." << std::endl
			      << message::unlock;
      for (int i = 0; i < _no_of_threads; ++i) {
	if (_threads[i].joinable()) {
	  _threads[i].join();
	}
      }
      MessageStreams::debug() << message::lock
			      << "... done" << std::endl
			      << message::unlock;
    }

    // finish the output, if necessary:
    if (_print_objects) {
      std::cout << "}" << std::endl
		<< message::unlock;
    }

    
    // finish the asymptote file, if necessary:
    if (CommandlineOptions::output_asy()) {
      Graphics::nodepos_to_asy();
      Graphics::define_draw_node_for_matrix_to_asy();
      Graphics::drawings_to_asy();
      Graphics::term_asy();
    }

    // finish the statistics file, if necessary:
    if (CommandlineOptions::output_stats()) {
      Statistics::term();
    }

    // destroy all objects that have been generated by this class:
    if (_root_nodeptr) {
      delete _root_nodeptr;
    }
    if (_switch_tableptr) {
      delete _switch_tableptr;
    }
    if (_classified_symmetriesptr) {
      delete _classified_symmetriesptr;
    }
#ifdef CONSTRUCTOR_DEBUG
    MessageStreams::debug() << message::lock
			    << "... done" << std::endl
			    << message::unlock;
#endif
  }

  // budgeting:
  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::update_node_budget() {
    _node_budget = std::max<size_type>(1, _open_nodes.size() * CommandlineOptions::scale_nodebudget() / 100);
    if (_node_budget < CommandlineOptions::min_nodebudget()) {
      _node_budget = CommandlineOptions::min_nodebudget();
    }
    else if (_node_budget > CommandlineOptions::max_nodebudget()) {
      _node_budget = CommandlineOptions::max_nodebudget();
    }
    if (CommandlineOptions::dump_status() && (_node_budget > CommandlineOptions::dump_frequency())) {
      _node_budget = CommandlineOptions::dump_frequency();
    }
  }

  // reporting:
  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline Message& SymmetricSubsetGraphMaster<mode>::report_progress(Message& msg, bool force) {
    if (force || _screenreport.decide_report(this->_nodecount)) {
      _screenreport.execute_master_report(msg,
					  this->get_no_of_idle_threads(),
					  this->_nodecount,
					  this->_symcount,
					  this->_totalcount,
					  this->_deadendcount,
					  this->_earlydeadendcount,
					  this->_open_nodes.size(),
					  CommandlineOptions::min_workbuffersize(),
					  this->_node_budget);
    }
    return msg;
  }

    
  // checkpointing:
  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::trigger_checkpoint(bool force) {
    if (force || _checkpoint.decide_dump(this->_nodecount)) {
      interrupt_workers();
      report_checkpoint();
      uninterrupt_workers();
    }
  }
  
  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::interrupt_workers() {
    _interrupt = true;      
    if (CommandlineOptions::parallel_enumeration()) {
      
      // parallel enumeration requires to sync all workers before reporting the checkpoint;
      // the following will cause the workers to stop at the next possible occasion and
      // the master will wait for all workers to get idle:
      while (_no_of_idle_threads < _no_of_threads) {
	{
	  std::unique_lock<std::mutex> main_lock(_main_waitmutex);
	  _main_condition.wait(main_lock, [this] { return wake_up(); } );
	}
	
	// collection of results is important after
	// any worker is done in order to change its state to idle:
	_collect_results();
      }
    } 
  }

  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::uninterrupt_workers() {
    _interrupt = false;
  }
  
  
  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline void SymmetricSubsetGraphMaster<mode>::report_checkpoint() {
    _checkpoint.open_dump();
    write(_checkpoint.dump_stream());
    _checkpoint.close_dump();
  }


  // shared data: none
  // callers: master thread only
  template <ssg_mode_type mode>
  inline const bool SymmetricSubsetGraphMaster<mode>::wake_up() const {

    // since this function is called inside the condition variable predicate,
    // we need to access the shared data lock-free (the lock is already held by the wait):
    return (_no_of_pending_results > 0) // need to collect results
      || (!_open_nodes.empty() && (_no_of_idle_threads > 0)) // can distribute work
      || (_open_nodes.empty() && (_no_of_idle_threads == _no_of_threads)); // need to terminate
  }

  // shared data: _no_of_idle_threads
  // caller: master thread only
  template <ssg_mode_type mode>
  inline const bool SymmetricSubsetGraphMaster<mode>::_exists_idle_thread() const {
    return (get_no_of_idle_threads() > 0);
  }

  // shared data: _no_of_pending_results
  // caller: master thread only
  template <ssg_mode_type mode>
  inline const bool SymmetricSubsetGraphMaster<mode>::_exists_pending_result() const {
    return (get_no_of_pending_results() > 0);
  }

  // shared data: _no_of_idle_threads
  // caller: master thread only
  template <ssg_mode_type mode>
  inline const bool SymmetricSubsetGraphMaster<mode>::_all_threads_idle() const {
    return (get_no_of_idle_threads() >= _no_of_threads);
  }

  // shared data: _no_of_pending_results
  // callers: master thread only
  template <ssg_mode_type mode>
  void SymmetricSubsetGraphMaster<mode>::_collect_results() {
    if (!_exists_pending_result()) {
      return;
    }
    for (int i = 0; i < _no_of_threads; ++i) {
      if (_workers[i].is_done()) {
	_nodecount                += _workers[i].nodecount();
	_totalcount               += _workers[i].totalcount();
	_symcount                 += _workers[i].symcount();
	_deadendcount             += _workers[i].deadendcount();
	_earlydeadendcount        += _workers[i].earlydeadendcount();
#ifdef COMPUTATIONS_DEBUG
	MessageStreams::debug() << message::lock
				<< "... copying remaning work into queue ..." << std::endl
				<< message::unlock;
#endif
	std::copy(_workers[i].open_nodes().begin(),
		  _workers[i].open_nodes().end(),
		  std::back_inserter<std::deque<node_type> >(_open_nodes));
	_current_workbuffersize = _open_nodes.size();
#ifdef COMPUTATIONS_DEBUG
	MessageStreams::debug() << message::lock
				<< "... done" << std::endl
				<< message::unlock;
#endif
	_workers[i].clear_results();
      }
    }
    _current_workbuffersize = _open_nodes.size();
    update_node_budget();
    report_progress(MessageStreams::verbose());
  }

  // shared data: _no_of_idle_threads, _no_of_pending_results
  // callers: master thread only
  template <ssg_mode_type mode>
  void SymmetricSubsetGraphMaster<mode>::_run() {
    if (CommandlineOptions::parallel_enumeration()) {
      
      // main loop of the master thread:
      set_no_of_idle_threads(_no_of_threads);
      set_no_of_pending_results(0);
      while (
	     !_open_nodes.empty() // need to distribute work
	     || !_all_threads_idle() // need to wait for results
	     || _exists_pending_result() // need to collect results
	     ) {

	// collect pending results of all done workers first:
	_collect_results();

	// check for termination by signal:
	if (Signal::signal_received()) {
	  trigger_checkpoint(true);
	  break;
	}

	// check for possible checkpoint if requested:
	if (CommandlineOptions::dump_status()) {
	  trigger_checkpoint(false);
	}

	// check for more work and idle threads:
	if (_exists_idle_thread() && !_open_nodes.empty()) {

	  // distribute work:
	  for (int i = 0; i < _no_of_threads; ++i) {
	    if (_workers[i].is_idle()) {
	      const node_type* next_nodeptr = new node_type(_open_nodes.front());
	      _open_nodes.pop_front();
	      _workers[i].pass_work(next_nodeptr);
	      if (_open_nodes.empty()) {
		break;
	      }
	    }
	  }

	  // notify all workers with assigned work:
	  for (int i = 0; i < _no_of_threads; ++i) {
	    if (_workers[i].is_hired()) {
	      _workers[i].worker_condition.notify_one();
	    }
	  }
	}

	// finally, wait if there is nothing to do:
	else {
	  while (!wake_up()) {
	    MessageStreams::debug() << message::lock
				    << "master is waiting for results || work & idle_thread || termination ..." << std::endl
				    << message::unlock;
	    std::unique_lock<std::mutex> main_lock(_main_waitmutex);
	    _main_condition.wait(main_lock, [this] { return wake_up(); } );
	  }
	}
      }
      MessageStreams::debug() << message::lock
			      << "work completed - master is terminating itself" << std::endl
			      << message::unlock;
    }
    else {
      
      if (CommandlineOptions::dump_status()) {

	// here, the node budget controls the interrupt for checkpointing:
	_node_budget = CommandlineOptions::dump_frequency();
      }
      else {
	
	// infinite node budget - one instance of SymmetricExtensionGraph
	// does the whole enumeration in one go:
	_node_budget = std::numeric_limits<size_type>::max();
      }

      size_type cnt = 0;
      while (!_open_nodes.empty()) {
	const node_type& next_node(_open_nodes.front());
	SymmetricSubsetGraph<mode> ssg(0,
				       0,
				       _no,
				       _rank,
				       _pointsptr,
				       _symmetriesptr,
				       _classified_symmetriesptr,
				       _switch_tableptr,
				       &next_node,
				       _open_nodes,
				       _result_set,
				       _print_objects,
				       _save_objects,
				       _node_budget,
				       &_current_workbuffersize,
				       &_interrupt);
	_nodecount                += ssg.nodecount();
	_totalcount               += ssg.totalcount();
	_symcount                 += ssg.symcount();
	_deadendcount             += ssg.deadendcount();
	_earlydeadendcount        += ssg.earlydeadendcount();
	_open_nodes.pop_front();
	_current_workbuffersize = _open_nodes.size();
	
	// check for termination by signal:
	if (Signal::signal_received()) {
	  trigger_checkpoint(true);
	  break;
	}
	report_progress(MessageStreams::verbose());
	if (CommandlineOptions::dump_status()) {
	  trigger_checkpoint(false);
	}
 	++cnt;
      }
    }
  }

  template <ssg_mode_type mode>
  void SymmetricSubsetGraphMaster<mode>::_init() {
    if (CommandlineOptions::use_classified_symmetries()) {
      MessageStreams::verbose() << message::lock
				<< "computing classified symmetries ..." << std::endl
				<< message::unlock;
      if (CommandlineOptions::parallel_symmetries()) {
	_classified_symmetriesptr = new ClassifiedSubsetSymmetries(std::min<int>(_no, CommandlineOptions::no_of_symtables()),
								   std::max<int>(CommandlineOptions::no_of_threads(), 1),
								   *_symmetriesptr);
      }
      else {
	_classified_symmetriesptr = new ClassifiedSubsetSymmetries(CommandlineOptions::no_of_symtables(),
								   std::min<int>(_no, CommandlineOptions::no_of_symtables()),
								   *_symmetriesptr);
      }
    }
    MessageStreams::verbose() << message::lock
			      << "... done." << std::endl
			      << message::unlock;
    if (CommandlineOptions::use_switch_tables()) {
      MessageStreams::verbose() << message::lock
				<< "computing switch table ..." << std::endl
				<< message::unlock;
      // _switch_tableptr = new SwitchTable<colexmax_mode>(*_symmetriesptr);
      _switch_tableptr = new SwitchTable<LabelSet, colexmax_mode>(_symmetriesptr->n(), _symmetriesptr->generators());
      MessageStreams::verbose() << message::lock
				<< "... done." << std::endl
				<< message::unlock;
    }
    
    // initialize the statistics file, if requested:
    if (CommandlineOptions::output_stats()) {
      Statistics::init();
    }
    
    // initialize the asymptote file, if requested and possible:
    if (CommandlineOptions::output_asy()) {
      Graphics::init_asy(CommandlineOptions::asy_file());
      Graphics::typedef_for_matrix_to_asy();
      Graphics::header_to_asy(_no, _rank);
      if (_rank == 3) {

	// in this case, we can make a drawing of the point configuration:
	Graphics::vertices_to_asy(Vertices(Circuits(Chirotope(*_pointsptr))));
	Graphics::points_to_asy(*_pointsptr);
      }
    }

    if (CommandlineOptions::parallel_enumeration()) {
      _no_of_threads = CommandlineOptions::no_of_threads();
      _init_workers();
    }
    else {

      // we need to generate a single worker for asymptote:
      if (CommandlineOptions::output_asy()) {
	Graphics::worker_to_asy();
      }
    }
    MessageStreams::verbose() << message::lock
			      << "generating root node with some symmetry-group preprocessing ..." << std::endl
			      << message::unlock;
    _root_nodeptr = new node_type(_symmetriesptr);
    if (mode == circuits) {
      MessageStreams::verbose() << message::lock
				<< "exploring all symmetry classes of circuits via extension ..." << std::endl
				<< message::unlock;
    }
    else {
      MessageStreams::verbose() << message::lock
				<< "exploring all symmetry classes of cocircuits via extension ..." << std::endl
				<< message::unlock;
    }
    set_no_of_idle_threads(_no_of_threads);
    set_no_of_pending_results(0);
    if (CommandlineOptions::read_status()) {
      
      // here, we read an intermediate state from a TOPCOM dump and initialize
      // _open_node from there using the _rootptr as a template containing
      // all the necessary global information in the nodes:
      std::ifstream read_str(CommandlineOptions::read_file());
      read(read_str);
      
      MessageStreams::verbose() << message::lock
				<< "SymmetricSubsetGraphMaster initialized from file "
				<< CommandlineOptions::read_file() << std::endl
				<< message::unlock;
      MessageStreams::debug() << message::lock
			      << "data:" 
			      << std::endl
			      << *this
			      << std::endl
			      << message::unlock;
      MessageStreams::verbose() << message::lock
				<< "checkpoint status:" << std::endl
				<< message::unlock;
      MessageStreams::verbose().print_hseparator();
      report_progress(MessageStreams::verbose(), true);
      MessageStreams::verbose().print_hseparator();
      MessageStreams::verbose() << std::endl
				<< message::unlock;
    }
    else {
      _open_nodes.push_back(*_root_nodeptr);
    }
    if (CommandlineOptions::output_stats()) {
      Statistics::report();
      Statistics::reset();
    }
    if (_print_objects) {
      MessageStreams::result() << _no << "," << _rank << ":\n{\n";
    }
  }

  template <ssg_mode_type mode>
  void SymmetricSubsetGraphMaster<mode>::_init_workers() {
    MessageStreams::verbose() << message::lock
			      << "init distributed workers ..." << std::endl
			      << message::unlock;
    for (int i = 0; i < _no_of_threads; ++i) {
      MessageStreams::verbose() << message::lock
				<< message::tab
				<< "initializing worker " << i << " ..." << std::endl
				<< message::unlock;
      _workers.emplace_back(i, *this);
      if (CommandlineOptions::output_asy()) {
	Graphics::worker_to_asy();
      }
      MessageStreams::verbose() << message::lock
				<< message::tab
				<< "... done" << std::endl
				<< message::unlock;
    }
    for (int i = 0; i < _no_of_threads; ++i) {
      MessageStreams::verbose() << message::lock
				<< message::tab
				<< "starting worker thread " << i 
				<< " ..." << std::endl
				<< message::unlock;
      _threads.push_back(std::thread(&SymmetricSubsetGraphMaster<mode>::Worker::operator(),
				     std::ref(_workers[i])));
      MessageStreams::verbose() << message::lock
				<< message::tab << "... done" << std::endl
				<< message::unlock;
    }
    MessageStreams::verbose() << message::lock
			      << "... done" << std::endl
			      << message::unlock;
  }

  
  // stream output/input:
  template <ssg_mode_type mode>
  std::ostream& SymmetricSubsetGraphMaster<mode>::write(std::ostream& ost) const {
    ost << _dump_startline << std::endl
	<< message::unlock;
    ost << "_dump_no                  " << _checkpoint.dump_no() << std::endl
	<< message::unlock;
    ost << "_totalcount               " << _totalcount << std::endl
	<< message::unlock;
    ost << "_symcount                 " << _symcount << std::endl
	<< message::unlock;
    ost << "_nodecount                " << _nodecount << std::endl
	<< message::unlock;
    ost << "_deadendcount             " << _deadendcount << std::endl
	<< message::unlock;
    ost << "_earlydeadendcount        " << _earlydeadendcount << std::endl
	<< message::unlock;
    ost << "_no                       " << _no << std::endl
	<< message::unlock;
    ost << "_rank                     " << _rank << std::endl
	<< message::unlock;
    if (_pointsptr) {
      ost << "_points                   " << *_pointsptr << std::endl
	  << message::unlock;
    }
    if (_chiroptr) {
      ost << "_chiro                    " << *_chiroptr << std::endl
	  << message::unlock;
    }
    if (!_symmetriesptr->generators().empty()) {
      ost << "_symmetry_generators      " << _symmetriesptr->generators() << std::endl
	  << message::unlock;
    }
    ost << "_current_workbuffersize   " << _current_workbuffersize << std::endl
	<< message::unlock;
    ost << "_open_nodes               " << _open_nodes << std::endl
	<< message::unlock;
    ost << _dump_endline << std::endl
	<< message::unlock;
    return ost;
  }

  template <ssg_mode_type mode>
  std::istream& SymmetricSubsetGraphMaster<mode>::read(std::istream& ist) {
    std::string dump_line;
    std::string last_dump_line;

    // first, set the flag if the standard-first-line was found to make sure the file is a TOPCOM dump:
    std::getline(ist, dump_line);
    if (_dump_startline.compare(0, _dump_startline.length(), dump_line) != 0) {
      MessageStreams::forced() << message::lock
			       << "SymmetricSubsetGraphMaster::read(std::istream& ist): not a TOPCOM dump file - exiting" << std::endl
			       << message::unlock;
      exit(1);
    }

    // first, check whether input file is a TOPCOM dump file:
    bool is_complete(false);
    
    while ((std::getline(ist, dump_line))) {
      std::string::size_type lastPos = dump_line.find_first_not_of(" ", 0);
      std::string::size_type pos     = dump_line.find_first_of(" ", lastPos);
      std::string keyword            = dump_line.substr(lastPos, pos - lastPos);
	  
      // next, parse the partial computational result from the dump file:
      if (keyword == "_totalcount") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());      
	std::istringstream istrst (value, std::ios::in);      
	if (!(istrst >> _totalcount)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _totalcount - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_totalcount initialized with " << _totalcount << std::endl
				<< message::unlock;
      }
      if (keyword == "_symcount") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	if (!(istrst >> _symcount)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _symcount - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_symcount initialized with " << _symcount << std::endl
				<< message::unlock;
      }
      if (keyword == "_nodecount") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	if (!(istrst >> _nodecount)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _nodecount - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_nodecount initialized with " << _nodecount << std::endl
				<< message::unlock;
      }
      if (keyword == "_deadendcount") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	if (!(istrst >> _deadendcount)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _deadendcount - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_deadendcount initialized with " << _deadendcount << std::endl
				<< message::unlock;
      }
      if (keyword == "_earlydeadendcount") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	if (!(istrst >> _earlydeadendcount)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _earlydeadendcount - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_earlydeadendcount initialized with " << _earlydeadendcount << std::endl
				<< message::unlock;
      }

      // next, some data is parsed that makes sure that the dumped computational results were
      // obtained with the "right" data:
      if (keyword == "_no") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	parameter_type no_check;
      
	if (!(istrst >> no_check)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _no - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	if (_no != no_check) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): no of points in input differs from no of points in dump file - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "no of points in input coincides with no of points in dump file: okay" << std::endl
				<< message::unlock;
      }
      if (keyword == "_rank") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	parameter_type rank_check;

	if (!(istrst >> rank_check)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _rank - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	if (_rank != rank_check) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): rank of input differs from rank in dump file - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "rank of input coincides with rank of dump file: okay" << std::endl
				<< message::unlock;
      }
      if (_pointsptr) {
	if (keyword == "_points") {
	  lastPos = dump_line.find_first_not_of(" ", pos);
	  std::string value = dump_line.substr(lastPos, dump_line.length());
	  std::istringstream istrst (value, std::ios::in);
	  PointConfiguration points_check;
	
	  if (!(istrst >> points_check)) {
	    MessageStreams::forced() << message::lock
				     << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _points - exiting" << std::endl
				     << message::unlock;
	    exit(1);
	  }
	  if (*_pointsptr != points_check) {
	    MessageStreams::forced() << message::lock
				     << "SymmetricSubsetGraphMaster::read(std::istream& ist): points of input differ from points in dump file - exiting" << std::endl
				     << message::unlock;
	    exit(1);
	  }
	  MessageStreams::debug() << message::lock
				  << "points of input coincide with points in dump file: okay" << std::endl
				  << message::unlock;
	}
      }
      if (_chiroptr) {
	if (keyword == "_chiro") {
	  lastPos = dump_line.find_first_not_of(" ", pos);
	  std::string value = dump_line.substr(lastPos, dump_line.length());
	  std::istringstream istrst (value, std::ios::in);
	  Chirotope chiro_check(*_pointsptr, false);
	
	  if (!(istrst >> chiro_check)) {
	    MessageStreams::forced() << message::lock
				     << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _chiro - exiting" << std::endl
				     << message::unlock;
	    exit(1);
	  }
	  if ((*_chiroptr) != chiro_check) {
	    MessageStreams::forced() << message::lock
				     << "SymmetricSubsetGraphMaster::read(std::istream& ist): chirotope of input differs from chirotope in dump file - exiting" << std::endl
				     << message::unlock;
	    exit(1);
	  }
	  MessageStreams::debug() << message::lock
				  << "chirotope of input coincides with chirotope of dump file: okay" << std::endl
				  << message::unlock;
	}
      }
      if (keyword == "_symmetry_generators") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	symmetry_collectordata symmetries_check;

	// we compare the generators only:
	if (!(istrst >> symmetries_check)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading symmetry generators - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	if (_symmetriesptr->generators() != symmetries_check) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): symmetries of input `"
				   << _symmetriesptr->generators() << "' differ from symmetries `"
				   << symmetries_check << "' in dump file - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "symmetries of input coincide with _symmetries in dump file: okay" << std::endl
				<< message::unlock;
      }

      // finally, read in the work buffer:
      if (keyword == "_current_workbuffersize") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);
	if (!(istrst >> _current_workbuffersize)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _current_workbuffersize - exiting" << std::endl
				   << message::unlock;
	  exit(1);
	}
	MessageStreams::debug() << message::lock
				<< "_current_workbuffersize initialized with " << _current_workbuffersize << std::endl
				<< message::unlock;
      }
      if (keyword == "_open_nodes") {
	lastPos = dump_line.find_first_not_of(" ", pos);
	std::string value = dump_line.substr(lastPos, dump_line.length());
	std::istringstream istrst (value, std::ios::in);

	// we read into an auxiliary structure because several pointers to
	// global data cannot be read but have to be constructed:
	std::deque<SymmetricSubsetGraphNodeReader<mode> > aux_reader;
	if (!(istrst >> aux_reader)) {
	  MessageStreams::forced() << message::lock
				   << "SymmetricSubsetGraphMaster::read(std::istream& ist): error while reading _open_nodes data into aux_reader - exiting"
				   << std::endl
				   << message::unlock;
	  exit(1);
	}

	// next, we construct the actual nodes into _open_nodes from aux_reader
	// utilizing that this class knows the global data:
	_open_nodes.clear();
	for (typename std::deque<SymmetricSubsetGraphNodeReader<mode> >::iterator iter = aux_reader.begin();
	     iter != aux_reader.end();
	     ++iter) {
	  _open_nodes.emplace_back(this->_symmetriesptr, // for the node
				   std::move(*iter));    // the data read into the aux reader
	}
      }
      last_dump_line = dump_line;
    }

    // finally, check last line for the standard end line in order
    // to make sure that the file is complete:
    if (_dump_endline.compare(0, _dump_endline.length(), last_dump_line) != 0) {
      MessageStreams::forced() << message::lock
			       << "last dump_line: " << last_dump_line << std::endl
			       << message::unlock;
      MessageStreams::forced() << message::lock
			       << "SymmetricSubsetGraphMaster::read(std::istream& ist): TOPCOM dump file is incomplete - exiting" << std::endl
			       << message::unlock;
      exit(1);
    }
    
    MessageStreams::verbose() << message::lock
			      << "... done" << std::endl
			      << message::unlock;
    return ist;
  }
  
}; // namespace topcom

#endif

// eof SymmetricSubsetGraphMaster.hh
