/*  
 *	Clustering algorithm for Proteinortho
 *	Reads edge list (blast-graph or abc/tsv) and splits connected components
 *	according to algebraic connectivity threshold
 *
 *	Last updated: 2023/05/15
 *	Author: Marcus Lechner, Paul Klemm
 */

#ifndef _PROTEINORTHOCLUSTERING
#define _PROTEINORTHOCLUSTERING

#include <iostream>
#include <fstream>
#include <sstream>
#include <string>
#include <list> //BFS/DFS
#include <map>
#include <algorithm>
#include <cmath>
#include <vector>
#include <stack>
#include <iomanip>
#include <cstdlib>
#include <ctime>
#include <memory>
#include <climits> // unsigned int max range
#include <functional>
#include <cstdint>
#include <cstdio>

// for the dynamic_queue
#include <thread>
#include <queue>
#include <mutex>
#include <condition_variable>

// for Smirnov-Grubbs test:
#include <complex>
#include <unordered_map>

using namespace std; // bad practive but i am used to it

#ifdef _OPENMP
#include <omp.h>
#endif

using namespace std;
#include <chrono>

// for sleep:
#ifdef _WIN32
#include <Windows.h>
#else
#include <unistd.h>
#endif

std::string getEnvVar( std::string const & key ){
	char * val = getenv( key.c_str() );
	return val == NULL ? std::string("") : std::string(val);
}

extern "C" {
	// For full documentation see https://netlib.org/lapack/explore-html/d2/d8a/group__double_s_yeigen_gaeed8a131adf56eaa2a9e5b1e0cce5718.html
	//(d|s)syevr LAPACK function
	//-      float (float/double)
	// --    symmetric 
	//   --- eigenvalue expert (more options, e.g. first k eigenvalues...)
	extern void ssyevr_( char* jobz, char* range, char* uplo, int* n, float* a,
				int* lda, float* vl, float* vu, int* il, int* iu, float* abstol,
				int* m, float* w, float* z, int* ldz, int* isuppz, float* work,
				int* lwork, int* iwork, int* liwork, int* info );
	extern void dsyevr_( char* jobz, char* range, char* uplo, int* n, double* a,
				int* lda, double* vl, double* vu, int* il, int* iu, double* abstol,
				int* m, double* w, double* z, int* ldz, int* isuppz, double* work,
				int* lwork, int* iwork, int* liwork, int* info );
	// (D/S)SYEVR computes selected eigenvalues and, optionally, eigenvectors
	// of a real symmetric matrix A.  Eigenvalues and eigenvectors can be
	// selected by specifying either a range of values or a range of
	// indices for the desired eigenvalues.
}

struct wedge {unsigned int edge; unsigned short weight=0;}; // edge = graph index (normally size_t)
struct protein {
	// edges contains the id of the graph entry that is adjacent to this entry, use it e.g. with adjacent_to_p = graph[ p.edges[0].edge ]
	vector<wedge> edges; 
	// species id, use the species map decode to string representation
	unsigned int species_id; 
	// full gene name as provided by the input blast graph (NOT the species name!)
	string full_name=""; 
	// a marker that is flipped if the node is printed as part of some orthogroup
	bool is_done=false;
};
class connectivity{
	public:

	double value; 
	bool param_min_species_is_true=false; // if this is true, then the average gene/species ratio is below the -minspecies param, therefore the connectivity is accepted regardless of its value
	// the previous solution with -connectivity can lead to unexpected behaviour (especially for small values => difficult to test on <0)

	connectivity(double val) : value(val){}
	connectivity(int val) : value(val){}
};

// Functions          
float string2float(string);
float string2double(string);
void tokenize(const string& , vector<string>& , const string&);
void parse_file(string);
void parse_abc_file(string);
void partition_graph(void);
void print_header(void);
void sort_species(void);
void stats(unsigned int,unsigned int,bool);
string getTime(void);
	bool param_verbose 		= true;
	bool param_core 		  = false;
	unsigned int param_prepare = 0;
	bool param_more = false;
	bool param_abc 		  = false;
	double param_con_threshold 	= 0.1;		// as a reference: a chain a-b-c-d has 0.25
	bool hidden_background=0;
	bool hidden_no_lollipop=0;
	unsigned int debug_level	= 0;
	float param_sep_purity 	= -1;		// as a reference: a-b-c will give +/-0.707107 and 2.34857e-08 
	unsigned int param_max_nodes	= 16777216; 
	bool param_max_nodes_was_set=false;
	float param_min_species	= 1;
	string param_rmgraph    = "remove.graph";
	unsigned int param_useWeights = 0; // 0=no, 1=yes if applicable, 2=always
	unsigned int param_minOpenmp = 256; // the minimum size of a for-loop for openmp to activate (openmp has some initialization costs)
	unsigned int param_coreMaxProteinsPerSpecies = 5;
	unsigned int param_coreMinSpecies = 0;
	double param_coreMinAlgCon = 0; // disabled, under development
	int param_useLapack = 1;

// min/max number of alg con iterations
const unsigned int min_iter = 16;			// below this value, results may vary
unsigned int param_max_iter = 8192;			// below this value, results may vary
float param_epsilon = 1e-8; // analog to http://people.sc.fsu.edu/~jburkardt/c_src/power_method/power_method_prb.c
int param_double = 0;
unsigned int param_max_nodes_weight = 1e+3;
unsigned int param_minnodes = 2;
float param_lapack_power_threshold_d = -1; // -1 = automatically detect according to the linear function of the nodesize and density

// Globals
unsigned int species_counter = 0;	// Species
unsigned int protein_counter = 0;	// Proteins
vector<string> species;			// Number -> Name
vector<protein> graph;			// Graph containing all protein data
unsigned int last_stat_lapack = 0;			// For progress stats
unsigned int last_stat_power = 0;			// For progress stats
bool last_stat_act = false;			// For progress stats
unsigned int edges = 0;			// number of edges
unsigned int job_i = 0;			
unsigned int job_n = 0;			
map<size_t,shared_ptr<ofstream> > proteinorthotmp_clean;
map<size_t,shared_ptr<ofstream> > tmp_debug;
map<size_t,shared_ptr<ofstream> > graph_clean;			// File to store graph data
vector<int> reorder_table;		// Tells how proteins/species must be sorted
unsigned int num_cpus=1;

// TMP Globals
map<string,int> species2id;		// Name -> Number
map<string,int> protein2id;		// Name -> Number

//TEST functions
bool test__max_of_diag();
bool test__generate_random_vector();
bool test__get_new_x();
bool test__makeOrthogonal();
bool test__normalize();
bool test__getY();
bool test__lapack_power(bool);
bool test__smirnovgrubb();

#ifdef DEBUG
	unsigned int total_number_of_iterations_convergence = 0;
	unsigned int total_number_of_kmere_calls = 0;
	void debug__graph_integrity(vector<unsigned int>&);
	void debug__print_edgelist (protein&, const unsigned int, const int);
	void debug__conn_integrity(vector<unsigned int>&, float);
	void debug__print_matrix( int m, int n, float* a, int lda );
#endif

class dynamic_queue{
	// adapted from: https://github.com/embeddedartistry/embedded-resources/blob/master/examples/cpp/dispatch.cpp
	// but heavily modified to the needs of proteinortho
	// the main idea:
	// 1. for "small" connected components (CC), use lapack, here all cores can work in parallel each working on one CC using lapack
	// 1.1. the part that searches the CC (BFS algorithm) is also one thread spawing all other jobs
	// 2. if a large CC is found, wait until all threads idle, then use one thread with all cores to work on the CC using the power iteration
	// side note: power is not very thread efficient, but it is better than nothing (idea for later: maybe 16 cores are sufficient, so modify the algorithm to use power with max 16 cores)

	typedef std::function<void(void)> fp_t; // function handler

	public:
		dynamic_queue(size_t thread_cnt = 1);
		~dynamic_queue();

		void start();
		void waitTilDone(); // wait until all threads are done (nothing in q_singleCore,q_allCores as well as nothing is computing)

		// dispatch and move
		void dispatch(fp_t&& op);
		void dispatch_allCores(fp_t&& op);
		void dispatch_log(string log);
		void dispatch_status(bool is_lapack);
		size_t pending();

		unsigned int status_count_lapack=0;
		unsigned int status_count_power=0;

		// Deleted operations
		dynamic_queue(const dynamic_queue& rhs) = delete;
		dynamic_queue& operator=(const dynamic_queue& rhs) = delete;
		dynamic_queue(dynamic_queue&& rhs) = delete;
		dynamic_queue& operator=(dynamic_queue&& rhs) = delete;

		std::vector<std::thread> threads;
		std::queue<fp_t> q_singleCore;
		std::queue<fp_t> q_allCores;
		std::map<size_t,bool> is_computing;
		std::mutex mut;
		// The mutex class is a synchronization primitive that can be used to protect shared data from being simultaneously accessed by multiple threads.
		// std::unique_lock<std::mutex> lock(mut); lock.lock() 
		// -> locks the mutex, blocks if the mutex is not available, use the .unlock() function to set free
		// the lock is automatically unlocked if it comes out of scope, therefore you see a lot of lock initialisation without an unlock

		std::condition_variable threadCommunicator;
		// this variable can notify threads to propagate updates (notify_all)

		bool quit_ = false; // flag to indicate that everything is done

		void dispatch_thread_handler(); // this is the main worker, one per thread
};

dynamic_queue::dynamic_queue(size_t thread_cnt) : threads(thread_cnt){}

void dynamic_queue::start(){
	if(debug_level>0)
		std::cerr << "Creating dispatch queue: threads:" << threads.size() << " q:" << q_singleCore.size() << "\n";
	for(size_t i = 0; i < threads.size(); i++){
		threads[i] = std::thread(&dynamic_queue::dispatch_thread_handler,this);
	}
	std::unique_lock<std::mutex> lock(mut);
	threadCommunicator.notify_all();
}

dynamic_queue::~dynamic_queue(){
	if(debug_level>0) std::cerr << getTime() << " Destructor: Destroying dispatch threads... threads:" << threads.size() << " q:" << q_singleCore.size() << "\n";

	// Signal to dispatch threads that it's time to wrap up
	std::unique_lock<std::mutex> lock(mut);
	quit_ = true;
	threadCommunicator.notify_all();
	lock.unlock();

	// Wait for threads to finish before we exit
	for(size_t i = 0; i < threads.size(); i++){ 
		if(threads[i].joinable()){ threads[i].join(); }
	}
	
	if(debug_level>0)
		cerr << getTime() << " ~dynamic_queue done, total count: lapack="<< status_count_lapack << " power=" << status_count_power << endl;
}

void dynamic_queue::waitTilDone(){
	while(true){
		bool all_idle=true;
		for (map<size_t,bool>::iterator it = is_computing.begin() ; it != is_computing.end(); it++) {
			if(it->second){ all_idle=false; break; }
		}
		if(all_idle && q_singleCore.size()==0 && q_allCores.size()==0){ break; }
		threadCommunicator.notify_all();
		sleep(1);
	}
}

size_t dynamic_queue::pending(){
	std::unique_lock<std::mutex> lock(mut); // unique_lock waits until a thread is ready to execute, gets "opened"/free at the end of the scope 
	return(q_singleCore.size());
}

void dynamic_queue::dispatch(fp_t&& op){
	std::unique_lock<std::mutex> lock(mut); // unique_lock waits until a thread is ready to execute, gets "opened"/free at the end of the scope 
	q_singleCore.push(std::move(op));
	threadCommunicator.notify_one();
}

void dynamic_queue::dispatch_log(string log){
	std::unique_lock<std::mutex> lock(mut);
	cerr << ( debug_level>0 ? "" : "\r" ) << log << ( debug_level>0 ? " {tid=" + to_string(std::hash<std::thread::id>{}(std::this_thread::get_id())) + "}" : "" ) << endl;
}

void dynamic_queue::dispatch_status(bool is_lapack){
	std::unique_lock<std::mutex> lock(mut);
	if(is_lapack) status_count_lapack++;
	else status_count_power++;
}

void dynamic_queue::dispatch_allCores(fp_t&& op){
	std::unique_lock<std::mutex> lock(mut);
	q_allCores.push(std::move(op));
	threadCommunicator.notify_all();
}

void dynamic_queue::dispatch_thread_handler(){
	size_t tid = std::hash<std::thread::id>{}(std::this_thread::get_id());
	do{
		std::unique_lock<std::mutex> lock(mut);
		// Wait until we have data or a quit signal
		threadCommunicator.wait(lock, [this] { 
			return ( 
				q_singleCore.size() || 
				q_allCores.size() || 
				quit_ 
			);
		});

		// after wait, we own the lock
		is_computing[tid]=false; // mark this thread as free

		// the normal queue q_singleCore has priority
		if( !quit_ && q_singleCore.size() ){
			is_computing[tid]=true;
			unsigned int active_cpus=0;
			for (map<size_t,bool>::iterator it=is_computing.begin() ; it != is_computing.end(); it++) { active_cpus+=it->second; }
			if(debug_level==0)
				stats(q_singleCore.size()+active_cpus,q_allCores.size(),1);
			auto op = std::move(q_singleCore.front());
			q_singleCore.pop();
			// unlock now that we're done messing with the queue
			lock.unlock();
			op(); // main work, here lapack / BFS / removeLowQualityEdges is invoked
			lock.lock();
			is_computing[tid]=false; // mark this thread as free
			lock.unlock();
		}else if( !quit_ && q_allCores.size() ){ // if there is no normal queue jobs, then do the all cores queue
			bool all_idle=true;
			for (map<size_t,bool>::iterator it=is_computing.begin() ; it != is_computing.end(); it++) { if(it->first != tid && it->second){all_idle=false;break;} }

			if( all_idle ){
				is_computing[tid]=true;
				if(debug_level==0)
					stats(q_singleCore.size(),q_allCores.size()+1,0);
				if (debug_level > 0)
					cerr << tid << " allcores,all_idle:"<< all_idle << endl;
				auto op = std::move(q_allCores.front());
				q_allCores.pop();
				// unlock now that we are done messing with the queue
				lock.unlock();
				op(); // main work, power iteration with all cores
				lock.lock();
				is_computing[tid]=false; // mark this thread as free
				lock.unlock();
			}
		}
	} while(!quit_);
}

///////////////////////////////////////////////////////////
// Main
///////////////////////////////////////////////////////////
void printHelp() {
	cerr << "proteinortho_clustering - Spectral partitioning algorithm (last updated with proteinortho v6.2.4)" << "\n";
	cerr << "-----------------------------------------------------" << "\n";
	cerr << "This tool is part of Proteinortho" << "\n";
	cerr << "" << "\n";
	cerr << "Usage:   proteinortho_clustering [OPTIONS] graph_files..." << "\n";
	cerr << "Options: -verbose          report progress" << "\n";
	cerr << "         -conn float       minimal connectivity: keep dissecting if algebraic connectivity is below (or if minspecies is satisfied). ["<<param_con_threshold<<"]\n                The higher this value the more strict the clustering is. For -conn 0 no splits are done only the connectivity values are calculated.\n                Use -conn -1 to just print out all connected components with no clustering (even without the connectivity calculation)" << "\n";
	cerr << "         -minspecies float stop clustering if ratio of genes/species is less or equal to minspecies regardless of the connectivity. A value of 1 correspond to a stop if there is exactly one gene per species present in a group. This overrules the -conn threshold. ["<<param_min_species<<"]" << "\n";
	cerr << "         -core             stop clustering if a split would result in groups that do not span across all species of the inital connected component (unless the connectivity is very low). This overrules the -conn threshold.\n";
	cerr << "         -rmgraph STRING   output file name for the graph" << "\n";
	cerr << "         -seed int         seed value for srand [current unix time]" << "\n";
	cerr << "         -lapack int       use the lapack package for the computation of the algebraic connectivity. 0=no, 1=yes if applicable, 2=always ["<<param_useLapack<<"]" << "\n";
	cerr << "         -cpus int         the number of threads used for openMP ["<<num_cpus<<"]" << "\n";
	cerr << "         -coreMaxProts int  the maximum number of proteins per species for -core ["<<param_coreMaxProteinsPerSpecies<<"]" << "\n";
	cerr << "         -coreMinSpecies int  the minimum number of species for -core ["<<param_coreMinSpecies<<"]" << "\n";
	cerr << "         -abc   flag to indicate the input is a abc formatted graph file instead of a blast-graph (tab-separated). Input is expected to be undirected. c is the similarity score (0-USHRT_MAX) e.g. the blast bitscore. For unwuighted use -weighted 0 or set c to 1." << "\n";
	cerr << "\ntechnical parameters:" << "\n";
	cerr << "         -test             various test-functions are called first [not set]" << "\n";
	cerr << "         -maxnodes int     only consider connected component with up to maxnodes nodes. If exceeded, remove outlying edges accoring to the Smirnov-Grubb test or greedily the worst 10 percent of edges (by weight) until satisfied. 0:disabled [default: min(max(5000,number of species * 5),1e+7)]" << "\n";
	cerr << "         -maxweight int    only use the edge weights for connected components with maxweight nodes ["<<param_max_nodes_weight<<"]" << "\n";
	cerr << "         -epsilon float    convergence threshold ["<<param_epsilon<<"]" << "\n";
	cerr << "         -weighted bool    the spectral partition is calculated using the bitscores. 0=no, 1=yes if applicable, 2=always ["<<param_useWeights<<"]" << "\n";
	cerr << "         -double int      use double precision. 0=no, 1=yes if applicable, 2=always ["<<param_double<<"]" << "\n";
	cerr << "         -minOpenmp int    the minimum number of nodes for parallel power iteration ["<<param_minOpenmp<<"]" << "\n";
	cerr << "         -powLapD | -power_d float	    the maximal graph density for the power iteration method, lapacks (d|s)syevr is used otherwise [adaptively choose optimal cutoff]" << "\n";
	cerr << "         -maxRunsConvergence int    the maximum number of runs for the calculation of the algebraic connectivity for the power iteration ["<<param_max_iter<<"]" << "\n";
	cerr << "         -more             output fiedler vector as well as the algebraic connectivity for each group" << "\n";
}
// deprecated: 	
// cerr << "         -ram int          maximal used ram threshold for LAPACK and the input graph in MB [16384]" << "\n";
// cerr << "         -kmere bool	    use the kmere-split heuristic ["<<param_useKmereHeuristic<<"]" << "\n";
// cerr << "         -purity float     threshold for purity: treat float values between -purity and purity as 0. -1=adaptively choose ["<<param_sep_purity<<"]" << "\n";

int main(int argc, char *argv[]) {

	if (argc <= 1) {
		printHelp();
		return 1;
	}

	try {
		#ifdef _OPENMP
			omp_set_dynamic(0);     // Explicitly disable dynamic teams
			// omp_set_num_threads(num_cpus); // deprecated, done by environment variable OMP_NUM_THREADS=1 for lapack and directly in the pragma loop for lapack (omp_set_num_threads does not work here...)
		#endif

		int rand_seed = 12345; //init randseed, fixed so results are more comparable

		// Read parameters
		int paras;
		vector<string> files;
		for (paras = 1; paras < argc; paras++) {
			string parameter = string(argv[paras]);
			if (parameter.substr(0, 1) != "-" || parameter == "-") {
				files.push_back(parameter);
			}
			else if (parameter == "-verbose") {
				paras++;
				if (string2float(string(argv[paras])) == 0) {
					param_verbose = false; 
				}
			}
			else if (parameter == "-core") {
				param_core = true;
			}
			else if (parameter == "-more") {
				param_more = true;
			}
			else if (parameter == "-abc") {
				param_abc = true;
			}
			else if (parameter == "-coreMaxProt" || parameter == "-coreMaxProts") {
				param_core = true;
				paras++;
				param_coreMaxProteinsPerSpecies = string2float(string(argv[paras]));
				if(param_coreMaxProteinsPerSpecies<=0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-prepare") {
				paras++;
				param_prepare = string2float(string(argv[paras]));
				if(param_prepare<=0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-coreMinSpecies") {
				param_core = true;
				paras++;
				param_coreMinSpecies = string2float(string(argv[paras]));
				if(param_coreMinSpecies<0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-coreMinAlgCon") {
				param_core = true;
				paras++;
				param_coreMinAlgCon = string2float(string(argv[paras]));
				if(param_coreMinAlgCon<0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-conn") {
				paras++;
				param_con_threshold = string2double(string(argv[paras]));
				if((param_con_threshold<0 && param_con_threshold!=-1) || param_con_threshold>1){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-background") { // hiden option, terminate after loading, no BFS, no clustering
				hidden_background = 1;
			}
			else if (parameter == "-nolollipop") { // hiden option, terminate after loading, no BFS, no clustering
				hidden_no_lollipop = 1;
			}
			else if (parameter == "-powLapD" || parameter == "-lapPowD" || parameter == "-power_d" || parameter == "-pld") {
				paras++;
				param_lapack_power_threshold_d = (string2float(string(argv[paras])));
			}
			else if(parameter == "-lapack"){
				paras++;
				param_useLapack = int(string2float(string(argv[paras])));
				if(param_useLapack!=0 && param_useLapack!=1 && param_useLapack!=2){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-maxnodes") {
				paras++;
				param_max_nodes = string2float(string(argv[paras]));
				param_max_nodes_was_set=true;
			}
			else if (parameter == "-ignoreminn") { // hidden option
				paras++;
				param_minnodes = string2float(string(argv[paras]));
			}
			else if (parameter == "-maxweight") {
				paras++;
				param_max_nodes_weight = string2float(string(argv[paras]));
			}
			else if (parameter == "-minnodes") {
				paras++;
				param_minnodes = string2float(string(argv[paras]));
			}
			else if (parameter == "-minspecies") {
				paras++;
				param_min_species = string2float(string(argv[paras]));
				if (param_min_species < 0) {cerr << string("-minspecies must at least be 0. Less than one gene per species is not possible as we only count those that have an entry.").c_str() << "\n";throw;}
			}
			else if (parameter == "-job") {
				paras++;
				vector<string> fields;
				tokenize(argv[paras], fields, "/");
				if (fields.size() != 2 || string2float(fields[0])<1 || string2float(fields[1])<1) {cerr << string("-job is invalid, please use the format -job=1/10 (starting with 1).").c_str() << "\n";throw;}
				job_i=string2float(fields[0]);
				job_n=string2float(fields[1]);
			}
			else if (parameter == "-debug") {
				paras++;
				debug_level = int(string2float(string(argv[paras])));
			}
			else if (parameter == "-epsilon") {
				paras++;
				param_epsilon = string2float(string(argv[paras]));
				if(param_epsilon<0||param_epsilon>1){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-double") {
				paras++;
				param_double = string2float(string(argv[paras]));
				if(param_double!=0 && param_double!=1 && param_double!=2){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}
			else if (parameter == "-minOpenmp") {
				paras++;
				param_minOpenmp = int(string2float(string(argv[paras])));
			}
			else if (parameter == "-weighted") {
				paras++;
				param_useWeights = int(string2float(string(argv[paras])));
			}
			else if (parameter == "-seed") {
				paras++;
				rand_seed = int(string2float(string(argv[paras])));
				if(rand_seed<0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
			}else if (parameter == "-rmgraph") {
				paras++;
				param_rmgraph = string(argv[paras]);
			}else if(parameter == "-cpus"){
				paras++;
				#ifdef _OPENMP
					if(int(string2float(string(argv[paras])))<0){cerr << string("Error: invalid value '"+string(argv[paras])+" for argument "+parameter+"'!").c_str() << "\n";throw;}
					omp_set_dynamic(0);     // Explicitly disable dynamic teams
					num_cpus=int(string2float(string(argv[paras])));
					// omp_set_num_threads(num_cpus); // deprecated, done by environment variable OMP_NUM_THREADS=1 for lapack and directly in the pragma loop for lapack (omp_set_num_threads does not work here...)
				#else
					cerr << "Error: missing openMP, please install/activate openMP if you want to use multiple cores.\n";
					throw;
				#endif
			}else if(parameter == "-maxRunsConvergence"){
				paras++;
				param_max_iter = int(string2float(string(argv[paras])));
			}else if(parameter == "-test"){
				bool test__max_of_diag_result = test__max_of_diag();
				bool test__generate_random_vector_result = test__generate_random_vector();
				bool test__get_new_x_result = test__get_new_x();
				bool test__makeOrthogonal_result = test__makeOrthogonal();
				bool test__normalize_result = test__normalize();
				bool test__getY_result = test__getY();
				bool test__lapack_result = test__lapack_power(true);
				bool test__power_result = test__lapack_power(false);
				bool test__smirnovgrubb_result = test__smirnovgrubb();
				cerr << "- test max_of_diag() : " << (test__max_of_diag_result ? "OK" : "Failed") << "\n";
				cerr << "- test generate_random_vector() : "<< (test__generate_random_vector_result? "OK" : "Failed") << "\n";
				cerr << "- test get_new_x() : " << (test__get_new_x_result? "OK" : "Failed") << "\n";
				cerr << "- test makeOrthogonal() : " << (test__makeOrthogonal_result? "OK" : "Failed") << "\n";
				cerr << "- test normalize() : " << (test__normalize_result? "OK" : "Failed") << "\n";
				cerr << "- test getY() : " << (test__getY_result? "OK" : "Failed") << "\n";
				cerr << "- test lapack() : " << (test__lapack_result? "OK" : "Failed") << "\n";
				cerr << "- test power() : " << (test__power_result? "OK" : "Failed") << "\n";
				cerr << "- test smirnov-grubb() : " << (test__smirnovgrubb_result? "OK" : "Failed") << "\n";
				if( !test__max_of_diag_result || !test__generate_random_vector_result || !test__get_new_x_result || !test__makeOrthogonal_result || !test__normalize_result || !test__getY_result || !test__lapack_result || !test__power_result || !test__smirnovgrubb_result ){ 
					cerr << string("Error: tests failed !").c_str() << "\n";throw;
				}else{
					cerr << "All test passed." << "\n";
					return 0;
				} 
			}
			else {
				printHelp();
				cerr << "\n" << "Sorry, unknown option '" << string(argv[paras]) << "'!" << "\n";
				return 1;
			}
		}

		srand(rand_seed);

		if(param_prepare>0){ param_con_threshold=-1; job_n=0; }

		if (debug_level > 0) cerr << getTime() << " [DEBUG]   Debug level " << debug_level << "\n";

		if(param_core){param_con_threshold=999;} // for -core the -conn parameter is disabled 

		if(getEnvVar("OMP_NUM_THREADS") != "1"){ // this is needed, otherwise this program will run with -cpus^2 threads ...
			// restart with OMP_NUM_THREADS=1
			string cmd="OMP_NUM_THREADS=1";
			for (paras = 0; paras < argc; paras++)
				cmd += " "+string(argv[paras]);
			return system(cmd.c_str());
		}

		// Parse files
		// if(param_abc){
		// 	species.push_back("input");
		// 	species2id["input"] = species_counter++;
		// }

		for (vector<string>::iterator it=files.begin() ; it != files.end(); it++) {

			if (debug_level > 0) cerr << getTime() << " [DEBUG]   Parsing file " << *it << "\n";
			if(param_abc){
				parse_abc_file(*it);
			}else{
				parse_file(*it);
			}
		
			if (debug_level > 0) cerr << getTime() << " [DEBUG]   I know " << (species_counter == 1 ? "no" : to_string(species_counter).c_str() ) <<  " species with " << protein_counter << " proteins and " << edges << " edges in sum" << "\n";
		}

		// Stats
		if (param_verbose) cerr << (species_counter == 1 ? "no" : to_string(species_counter).c_str()) << " species" << "\n" << protein_counter << " paired proteins" << "\n" << edges << " bidirectional edges" << "\n";

		if( !param_max_nodes_was_set ){ param_max_nodes = species_counter*5; if(param_max_nodes<5000){param_max_nodes=5000;} if(param_useLapack ==2 && param_max_nodes>32768){param_max_nodes=32768;} if(param_max_nodes>1e+7){param_max_nodes=1e+7;} }
		if((param_useLapack == 2 || param_lapack_power_threshold_d == 0) && param_max_nodes>32768){param_max_nodes=32768;} // upper limit for lapack

		if (debug_level > 0) cerr << getTime() << " [DEBUG]   Maximumum number of nodes for connectivity calculations is " << param_max_nodes << "\n";

		// Prepare sort of output
		if (debug_level > 0) cerr << getTime() << " [DEBUG]   Sorting known species" << "\n";
		sort_species();

		// Write output header
		print_header();

		if(hidden_background){int r = system(("touch '"+param_rmgraph+".done'").c_str());return 0;}			

		// Clustering
		partition_graph();

		if(debug_level>0) cerr << getTime() << "[DEBUG] done with clustering" << endl;

		if(graph.size() == 0){return 0;}

		// concat the remove graph output files
		ofstream OFS((param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")).c_str());
		for(map<size_t,shared_ptr<ofstream> > ::iterator it = graph_clean.begin() ; it != graph_clean.end() ; ++it){
			it->second->close();
			ifstream IFS((param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(it->first)).c_str());
			if (IFS.is_open()) {
			while (!IFS.eof()) {
					string line;
					getline(IFS, line);
					if(line != "")
						OFS << line << "\n";
				}
			}
			IFS.close();
			// unlink tmp file
			int r = system(("rm '"+param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(it->first)+"'").c_str());
		}
		OFS.close();

		if(debug_level>0) cerr << getTime() << "[DEBUG] done <ofstream>graph_clean" << endl;

		for(map<size_t,shared_ptr<ofstream> > ::iterator it = proteinorthotmp_clean.begin() ; it != proteinorthotmp_clean.end() ; ++it){
			it->second->close(); // close ofstream
			// dump to STDOUT
			ifstream IFS((param_rmgraph+"_proteinortho_tmp_"+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(it->first)).c_str());
			if (IFS.is_open()) {
			while (!IFS.eof()) {
					string line;
					getline(IFS, line);
					if(line != "")
						cout << line << "\n";
				}
			}
			IFS.close();
			// unlink tmp file
			int r = system(("rm '"+param_rmgraph+"_proteinortho_tmp_"+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(it->first)+"'").c_str());
		}

		if(debug_level>0) cerr << getTime() << "[DEBUG] done <ofstream>proteinorthotmp_clean" << endl;

		for(map<size_t,shared_ptr<ofstream> > ::iterator it = tmp_debug.begin() ; it != tmp_debug.end() ; ++it){
			it->second->close(); // close ofstream
		}
		
		int r = system(("touch '"+param_rmgraph+".done'").c_str()); // this catches the case where a core dump is thrown at the return 0 (but everything is fine), this error is rare but can occure (out of the 42k CC of hannah, in ~10 cases a core dump is thrown at the return below and everything is fine)
		return 0;
	}
	catch(string& error) {
		cerr << "[ERROR]   " << error << "\n";
		return 1;
	}
}

string getCCid(vector<unsigned int> nodes){
	unsigned int n = nodes.size();
	double id=0;
	if(n==0){return "?";}
	map<unsigned int,unsigned int> mapping;
	for (unsigned int i = 0; i < (unsigned int)n; i++) {mapping[nodes[i]] = i;}
	for (unsigned int i = 0 ; i < (unsigned int)n ; i++){
		unsigned int from = nodes[i]; 
		if(!mapping.count(from)){continue;}
		for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
			unsigned int to = graph[from].edges[j].edge;
			if(!mapping.count(to)){continue;}
			//id+=to_string(from)+"-"+to_string(graph[from].edges[j].edge)+":"+to_string(graph[from].edges[j].weight)+",";
			id+=(from+1)*(graph[from].edges[j].edge+1)*(graph[from].edges[j].weight+1);
		}
	}
	return to_string((unsigned int)id);
}

unsigned int getCCid_uint(unsigned int x) {
	// hash number to a large uint
    x = ((x >> 16) ^ x) * 0x45d9f3b;
    x = ((x >> 16) ^ x) * 0x45d9f3b;
    x = (x >> 16) ^ x;
    // extract the first 4 digits
    std::string strNum = std::to_string(x);
    if (strNum.length() <= 4) { return x; }
    std::string strFirstFourDigits = strNum.substr(0, 4);
    unsigned int firstFourDigits = static_cast<unsigned int>(std::stoi(strFirstFourDigits));
    return firstFourDigits;
}

class ConnectedComponent // A graph representation (as vector of idx of the induced subgraph of 'graph') with some graph attributes (graph density, sum of node degrees)
{
	public:
	vector<unsigned int> m_content_CC; //ids of the induced subgraph of graph
	unsigned int d_sum; // sum of node degrees 
	double density;
	unsigned int species_num;
	vector<float> x_hat; // optional eigenvector (used in -more)

	ConnectedComponent(){
		d_sum=0;
		density=-1; // mark for calculation !
		species_num=0;
	}

	void calc_dsum(){
		if(param_con_threshold<0){return;}
		d_sum=0;
		map<unsigned int,bool> m_content_CC_set;
		for (unsigned int i = 0; i < m_content_CC.size(); i++) { m_content_CC_set[m_content_CC[i]]=true; }
		for (unsigned int i = 0; i < m_content_CC.size(); i++) {
			for (unsigned int j = 0; j < graph[m_content_CC[i]].edges.size(); j++) {
				if(m_content_CC_set.count(graph[m_content_CC[i]].edges[j].edge)){
					d_sum+=1;
				}
			}
		}
	}
	void calc_density(){
		if(param_con_threshold<0){density=1;return;}
		density=size()>1 ? (((double)d_sum))/(((double)(((double)size()-1.0)*(double)size()))) : 1;
	}
	
	unsigned int& operator[](unsigned int i){
		if(i > m_content_CC.size()){cerr << "[CRITICAL ERROR] out of bound in ConnectedComponent[]" << "\n"; throw;}
		return m_content_CC[i];
	}
	const unsigned int& operator[](unsigned int i)const{
		if(i > m_content_CC.size()){cerr << "[CRITICAL ERROR] out of bound in ConnectedComponent[]" << "\n"; throw;} 
		return m_content_CC[i];
	}

	unsigned int size(){ return m_content_CC.size();}
	unsigned int size()const { return m_content_CC.size();}

	void operator = (const ConnectedComponent &D ) { 
		m_content_CC = D.m_content_CC;
		d_sum = D.d_sum;
		density = D.density;
		species_num = D.species_num;
	}
	void push_back(unsigned int i) { 
		m_content_CC.push_back(i);
	}
};

struct partition_status {
    unsigned int restarted = 0;
    bool lollipop = false;
    bool use_all_cores = false;
    // bool did_grubbs_because_bad_split = false;
    //unsigned int bad_split = 0;
    bool do_lapack = true;
};
connectivity getConnectivity_float(vector<unsigned int>*,bool,vector<float>*,bool,dynamic_queue *);
connectivity getConnectivity_double(vector<unsigned int>*,bool,vector<double>*,bool,dynamic_queue *);
void partition_CC(ConnectedComponent, dynamic_queue *, partition_status);
void find_CCs(dynamic_queue*);
void find_CCs_givenNodes(dynamic_queue*,vector<unsigned int>);
void print_group(ConnectedComponent& , connectivity, size_t, bool, dynamic_queue *);
float calc_group(vector<unsigned int>*);

ConnectedComponent BFS( map<unsigned int, bool> * done, unsigned int cur_node , float cut_off ){

	/*
	*
	* Simple BFS implementation
	* done map is used to identify allready visited nodes (can be modified before function call to indicate forbidden nodes)
	* cut_off = ignore all edges below this cut off
	* 
	*/
	size_t tid=std::hash<std::thread::id>{}(std::this_thread::get_id());
	if(!graph_clean.count(tid))
		graph_clean[tid]=make_shared<ofstream>((param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(tid)).c_str());

	ConnectedComponent ret; //return vector
	list<unsigned int> q;
	q.push_back(cur_node); // the todo list
	(*done)[cur_node]=true;

	while(q.size()>0){

		list<unsigned int> q_new;

		// iterate over all the nodes of q
		// add adjacent nodes of this node to q_new
		// mark as done

		for(list<unsigned int>::iterator it = q.begin() ; it != q.end() ; ++it){
			cur_node = *it;

			if ( graph[cur_node].is_done ){continue;}

			ret.push_back(cur_node);

			// count the adjacent edges that are not in done :
			for (unsigned int j = 0; j < graph[cur_node].edges.size(); j++) {
				if(graph[graph[cur_node].edges[j].edge].is_done){continue;}
				unsigned int adjacency_node = graph[cur_node].edges[j].edge;
				if( !done->count(adjacency_node) || !(*done)[adjacency_node] ){ ret.d_sum++; }
			}

			(*done)[cur_node] = true;

			if(graph[cur_node].is_done){continue;}

			for (unsigned int j = 0; j < graph[cur_node].edges.size(); j++) {

				if(graph[graph[cur_node].edges[j].edge].is_done){continue;}

				if(graph[cur_node].edges[j].weight < cut_off){
					protein node_i = graph[cur_node];
					protein node_j = graph[node_i.edges[j].edge];
					(*graph_clean[tid]) << node_i.full_name << "\t" << species[node_i.species_id] << "\t" << node_j.full_name << "\t" << species[node_j.species_id] << "\n";
					continue;
				} // ignore

				unsigned int adjacency_node = graph[cur_node].edges[j].edge;

				if(adjacency_node > graph.size()){
					cerr << string("[ERROR] : Input graph is invalid. The node "+graph[cur_node].full_name +" is reporting an edge/adjacent node, that is not present in the graph.").c_str() << "\n";throw;
				}

				if( !done->count(adjacency_node) || !(*done)[adjacency_node] ){
					(*done)[adjacency_node] = true;
					q_new.push_back(adjacency_node);
				}
			}
		}

		q=q_new;
	}
	return ret;
}

struct compare_ConnectedComponents { //sort from large to small
	bool operator() (const ConnectedComponent &a, const ConnectedComponent &b) const {
		return a.density < b.density;
	}
};

// critical values for alpha=0.01 from https://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm
// vector index corresponds to the freedom degree, i.e. idx:0 => f:1, 1=>2, ..., 99=>100, idx:100 => f:inf
const double t_distribution_crit_vals[] = { 318.313,22.327,10.215,7.173,5.893,5.208,4.782,4.499,4.296,4.143,4.024,3.929,3.852,3.787,3.733,3.686,3.646,3.610,3.579,3.552,3.527,3.505,3.485,3.467,3.450,3.435,3.421,3.408,3.396,3.385,3.375,3.365,3.356,3.348,3.340,3.333,3.326,3.319,3.313,3.307,3.301,3.296,3.291,3.286,3.281,3.277,3.273,3.269,3.265,3.261,3.258,3.255,3.251,3.248,3.245,3.242,3.239,3.237,3.234,3.232,3.229,3.227,3.225,3.223,3.220,3.218,3.216,3.214,3.213,3.211,3.209,3.207,3.206,3.204,3.202,3.201,3.199,3.198,3.197,3.195,3.194,3.193,3.191,3.190,3.189,3.188,3.187,3.185,3.184,3.183,3.182,3.181,3.180,3.179,3.178,3.177,3.176,3.175,3.175,3.174,3.090 };

double meanOfData(vector<double>& data, unsigned int index){
	double sumval = 0;
	double n = (double) data.size() - (double)(index+1);
	for(unsigned int i = 0; i<data.size(); ++i){ if(i<=index){continue;} sumval += data[i]; }
	return sumval/n;
}

double varianceOfData(vector<double>& data, unsigned int index){
	double size = (double) data.size() - (double)(index+1);
	double mean = meanOfData(data,index);
	double var = 0;
	for(unsigned int i = 0; i<data.size(); ++i){ if(i<=index){continue;} var += (((data[i]-mean)*( data[i]-mean))/size); }
	return var;
}

double lowerSmirnovGrubbsTest_binary_search_cutoff(vector<double> data) { // v6.2.3
	// input : vector of edge weights 
	// output : the lowest edge weight s.t. all values above are without outliers (H0 of lowerSmirnovGrubbsTest)
	// 
	// sort the data, then we can just push a pointer 
	//    that marks all lower values as outliers that are removed 
	//    and the others as the dataset to test.
	// This is way faster than removing the values one-by-one but there is no guarantee on convergence!
	// 
	// search only in the lowest 2/3 of the data.
	// start in the middle at 1/3.
	// iteratively adjust the index depending on the test statistic H0 or H1:
	// for H0 : no outlier detected, this means we need to lower the index to search for outlier
	// for H1 : we found outlier (at least one), so we can increase the index (mark those as removed) until H0 occurs again
	// adjust the index in a binary search style.
	// 
	// note: alpha is 0.01

	sort(data.begin(), data.end()); // sort data

	map<unsigned int,bool> outliers; // mark elements as outliers, those are just ignored (instead of removing them from data)

	unsigned int index = data.size()/3;
	unsigned int upper = data.size()*2/3;
	unsigned int last_index = 0;
	unsigned int number_iterations = 0;
	while( number_iterations++ < 5000 ) { // maximal iterations, keep in mind this is only used for very large connected components

		// calculate the test statistics of this value
		double test_statistic = abs(data[index] - meanOfData(data,index)) / sqrt(varianceOfData(data,index)); 
		
		// calculate the reference value to test against
		unsigned int n = data.size() - (index+1);
		double t = t_distribution_crit_vals[ n-2-1 > 100 ? 100 : n-2-1 ]; // -2 for the degrees of freedom and -1 as the index starts at 0 corresponding to freedom 1, id 100 corresponds to inf t-distribution
		t=t*t; // square beforehand to save time
		double g_test = (((double)n-1) / sqrt((double)n)) * (sqrt( t / ((double)n-2 + t)));

		// test H0/H1
		if( test_statistic > g_test ){ 
			// => the test statistic is significant => H1 got accepted => there are outliers increase the border 
			index = index + (upper-index)/2;
		}else{
			// H0 => no outlier => decrease the border to search for the outlier
			upper = index;
			index = index/2; 
		}
		if(index == last_index || upper - index < 1){break;}
		last_index = index;
	}
	return data[index];
}

void removeLowQualityEdges( ConnectedComponent cur_cc , dynamic_queue *q , bool comes_from_main_call ){
	/*
	 * remove low quality edges
	 * 
	 * 1. try the binary search Smirnov-Grubbs test (remove lowest edge bitscore until H0:no outlier)
	 * 1.1 if the cluster did decompose into smaller clusters -> done
	 * 
	 * 2. as fallback, remove the 10% edges with lowest bitscores iteratively 
	 * 2.1 find the range of values of this CC -> define a cut-off = cut_off=min_w+0.1*(max_w-min_w);
	 * 2.2 remove all edges below that value
	 * 2.3 redo BFS and start over again for this cluster until -maxnodes are satisfied
	 * 
	 */
	double p=1e-5;

	string id_cur_cc="";
	if(debug_level>0) id_cur_cc=getCCid(cur_cc.m_content_CC);
	if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges size=" + to_string(cur_cc.size()) + " start");

	map<unsigned int, bool> cur_cc_map;
	for (unsigned int i = 0; i < cur_cc.size(); i++) {
		cur_cc_map[cur_cc[i]]=1;
	}

	// find the lowest 10% of edge values (min+p*(max-min))
	float min_w = -1; 
	float max_w = -1;
	vector<double> data;
	unsigned int original_number_nodes=cur_cc.size();
	for (unsigned int i = 0; i < cur_cc.size(); i++) {
		unsigned int from=cur_cc[i];
		for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
			unsigned int to = graph[from].edges[j].edge;
			if(!cur_cc_map.count(to)){continue;}
			unsigned int weight = graph[from].edges[j].weight;
			data.push_back( weight<=0 ? 0 : log((double)weight) ); // log weights s.t. they are more normally distributed ...
			if(min_w == -1 || weight < min_w){min_w=weight;} 
			if(min_w == -1 || weight > max_w){max_w=weight;}
		}
	}
	float cut_off = (float)exp(lowerSmirnovGrubbsTest_binary_search_cutoff(data)); // transform back

	if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+"  removeLowQualityEdges:ini size=" + to_string(cur_cc.size()) + " cut_off(lowerSmirnovGrubbsTest_binary_search_cutoff)=" + to_string(cut_off) + " fallback="+to_string(min_w+0.1*(max_w-min_w))+ " min_w=" +to_string(min_w) + " max_w=" + to_string(max_w));

	// fallback use the lowest 10%
	if(cut_off<=0){ cut_off=min_w+cut_off*(max_w-min_w); }

	if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+"  removeLowQualityEdges:ini2 size=" + to_string(cur_cc.size()) + " cut_off=" + to_string(cut_off) + " min_w=" +to_string(min_w) + " max_w=" + to_string(max_w));

	vector<ConnectedComponent> todoCCs; // gather CC that are still too large here directly.
	// the recursive solution did lead to a core dump (probably max nested function call)
	// additionally the function did collapse when using one core and was only usable with all cores, now it is fine with one core again !
	todoCCs.push_back(cur_cc);

	unsigned int it=0;
	while( todoCCs.size() >0 ){ 
		it++;

		vector<ConnectedComponent> newtodoCCs;

		for (unsigned int i = 0; i < todoCCs.size(); ++i){

			ConnectedComponent *work_cc = &todoCCs[i];

			map<unsigned int, bool> done;	// Keep track on what was done (for each node)
			// find all nodes outside todo_work and remove them from the done vector !
			for (unsigned int i = 0 ; i < (*work_cc).size() ; i++) {
				unsigned int from = (*work_cc)[i]; 
				done[ from ] = 0;
			}
			for (unsigned int i = 0 ; i < (*work_cc).size() ; i++) {
				unsigned int from = (*work_cc)[i]; 
				for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
					if(!done.count(graph[from].edges[j].edge))
						done[ graph[from].edges[j].edge ]=1;
				}
			}

			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges size=" + to_string((*work_cc).size()) + " while ");
			
			for (unsigned int id = 0 ; id < (*work_cc).size() ; id++) {
			
				unsigned int protein_id = (*work_cc)[id];

				if ( (done.count(protein_id) && done[protein_id]) || graph[protein_id].is_done){continue;}// We were here already

				if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc + "   removeLowQualityEdges BFS start" );

				done[protein_id]=true; // mark this node
				ConnectedComponent sub_work_cc = BFS(&done,protein_id,cut_off); // get the CC of the current node (protein_id) 

				if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+"  removeLowQualityEdges intial_cc.size=" + to_string((*work_cc).size()) + " find_CCs:start at " + to_string( protein_id ) + " found CC of sub_work_cc.size=" + to_string( sub_work_cc.size() ));

				if (( param_max_nodes> 0 && sub_work_cc.size() > param_max_nodes) || sub_work_cc.size() == cur_cc.size() ) {
					// the current CC is still too large / nothing did change
					// add back to todo
					newtodoCCs.push_back(sub_work_cc);
					if(debug_level>0 && param_max_nodes>0 && sub_work_cc.size() > param_max_nodes) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+" removeLowQualityEdges Found a very large connected component that contains " + to_string(sub_work_cc.size()) + ">" + to_string(param_max_nodes) +" (maxnodes) elements. For this component the fast but coarse heuristic removeLowQualityEdges will be used (involving the Smirnov-Grubb outlier test).");
					continue;
				}

				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+ " :removeLowQualityEdges calc_dsum,calc_density ");
				sub_work_cc.calc_dsum();
				sub_work_cc.calc_density();
				if(sub_work_cc.density > 1){ cerr << "[WARNING] : The input graph has duplicated edges, this lead to an invalid graph density of " << sub_work_cc.density << " (should be <1). Please clean the .blast-graph with 'proteinortho.pl --cleanblast --step=3 --project=...' or use the cleanupblastgraph tool in src/ to remove the duplicated edges." << "\n"; }
				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+ " :removeLowQualityEdges Found connected component: " +to_string(sub_work_cc.size())+ " proteins (ID: "+ to_string(protein_id) + "), graph density=" + to_string( sub_work_cc.density )+ ", sum of degrees=" + to_string(sub_work_cc.d_sum)+ " ini from " + graph[protein_id].full_name + " of size=" + to_string(sub_work_cc.size()));

				 // here did_grubbs_because_bad_split=true is not needed as this is a new CC, so it should be able to trigger this fallback again
				if(sub_work_cc.size()>1){
					if(job_n>0){ unsigned int id = getCCid_uint(sub_work_cc.m_content_CC[0]); if( id % job_n != job_i - 1 ){ continue; } }
					q->dispatch([sub_work_cc,q]{ partition_status status; partition_CC(sub_work_cc,q,status); });
				} // dont report singles
			}
		}

		if(newtodoCCs.size()>0){	

			if(it % 5 == 0 && p < 0.2){ 
				if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges incrementing p=" + to_string(p) + " to "+to_string(p*2));
				p *= 2; 
			}

			// still there are some left, increase the cur_off value 
			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges incrementing cut_off=" + to_string(cut_off) + " to "+to_string(cut_off+(max_w-min_w == 0 ? 10 : p*(max_w-min_w))));
			cut_off+=(max_w-min_w == 0 ? 10 : p*(max_w-min_w));

			// then start over
		}

		todoCCs=newtodoCCs;
	}

	if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"  removeLowQualityEdges size=" + to_string( cur_cc.size() ));
}

void removeLollipopSubGraph_and_dispatch_allCores( ConnectedComponent cur_cc , dynamic_queue *q , partition_status status ){
	/* v6.2.4
	 * remove Lollipop / long branches
	 * in linear time collect all sub-graphs that have a degree <= 2
	 * if the lollipop is of length 3 or higher -> remove and add back to normal queue
	 * the remaining graph is added to the allCores queue
	 */
	unsigned int min_lollipop_length=3; 

	size_t tid=std::hash<std::thread::id>{}(std::this_thread::get_id());
	if(!graph_clean.count(tid))
		graph_clean[tid]=make_shared<ofstream>((param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(tid)).c_str());

	if( !hidden_no_lollipop && ! status.lollipop && cur_cc.size()>500 ){ // safeguard

		string id_cur_cc="";
		if(debug_level>0) id_cur_cc=getCCid(cur_cc.m_content_CC);
		if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores size=" + to_string(cur_cc.size()) + " start");

		map<unsigned int, unsigned int> node2lollipop;
		map<unsigned int, vector<unsigned int> > lollipop2nodes;
		unsigned int lollipop_i=0;
		map<unsigned int, unsigned int> degree; // this serves 2 purposes: 1. degree of nodes only within the cur_cc, 2. test if a node is in cur_cc with .count()
		queue<unsigned int> todo;
		for (unsigned int i = 0; i < cur_cc.size(); i++) {
			unsigned int from=cur_cc[i];
			degree[from]=0;
		}
		for (unsigned int i = 0; i < cur_cc.size(); i++) {
			unsigned int from=cur_cc[i];
			for (unsigned int i = 0; i < graph[from].edges.size(); ++i){
				if( degree.count( graph[from].edges[i].edge ) ){degree[from]++;} // only of nodes within cur_cc 
			}
		}

		// collect the end-points (nodes of degree 1) as starting points
		for (unsigned int i = 0; i < cur_cc.size(); i++) {
			unsigned int from=cur_cc[i];
			if(degree[from] == 1){
				todo.push(from);
				node2lollipop[from]=lollipop_i;
				lollipop2nodes[lollipop_i].push_back(from);
				lollipop_i++;
			}
		}
		
		if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores todo=" + to_string(todo.size()) + "");

		while(todo.size()>0){

			unsigned int from=todo.front();
			todo.pop();

			if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores from=" + to_string(from) + "");

			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
				unsigned int to = graph[from].edges[j].edge;

				if( !degree.count(to) || degree[ to ] > 2 ){continue;}

				if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores  to=" + to_string(to) + "");

				if(node2lollipop.count(to) && node2lollipop[to] != node2lollipop[from]){
					lollipop2nodes[ node2lollipop[from] ].insert(lollipop2nodes[ node2lollipop[from] ].end(), lollipop2nodes[ node2lollipop[to] ].begin(), lollipop2nodes[ node2lollipop[to] ].end());
					map<unsigned int, vector<unsigned int> >::iterator iter = lollipop2nodes.find( node2lollipop[to] ) ;
					lollipop2nodes.erase( iter );
					for (map<unsigned int, unsigned int>::iterator it = node2lollipop.begin(); it != node2lollipop.end();++it){
						if(it->second == node2lollipop[to]){
							node2lollipop[it->first]=node2lollipop[from];
						}
					}
				}else if(!node2lollipop.count(to)){
					node2lollipop[ to ]=node2lollipop[ from ];
					lollipop2nodes[ node2lollipop[from] ].push_back( to );
					todo.push(to);
				}
			}
		}

		if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores lollipop2nodes=" + to_string(lollipop2nodes.size()) + "");

		bool did_cut_lollipop=false;
		for (unsigned int i = 0; i < lollipop_i; ++i){

			if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores lollipop2nodes["+to_string(i)+"]=" + to_string(lollipop2nodes[i].size()) + "");

			if( lollipop2nodes [ i ].size() >= min_lollipop_length ){

				if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores min_lollipop_length!");

				ConnectedComponent lollipop_CC;
				for (unsigned int j = 0; j < lollipop2nodes [ i ].size(); ++j){
					lollipop_CC.push_back( lollipop2nodes [ i ][ j ] );
					if(debug_level>2) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLollipopSubGraph_and_dispatch_allCores lollipop2nodes["+to_string(i)+"]=>" + ( graph[ lollipop2nodes [ i ][ j ] ].full_name ) + "");
				}
				if(debug_level>2) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"  removeLollipopSubGraph_and_dispatch_allCores lollipop-size=" + to_string( lollipop_CC.size() ));

				did_cut_lollipop=true;
				q->dispatch([lollipop_CC,q]{ partition_status status; status.lollipop=true; partition_CC(lollipop_CC,q,status); });

				// add the edges that connect the lollipop to the rest to the remove.graph output !
				// the last node added to the lollipop is the one that is connected to the rest of the graph

				unsigned int pre_last_node = lollipop2nodes [ i ] [ lollipop2nodes [ i ].size()-2 ];
				unsigned int last_node = lollipop2nodes [ i ] [ lollipop2nodes [ i ].size()-1 ];
				unsigned int other_node = 0;
				bool set_other_node=false;

				bool is_uninteresting=true;
				for(unsigned int j = 0 ; j < graph[last_node].edges.size() ; j++){ 
					if( ! node2lollipop[ graph[last_node].edges[j].edge ] && 
						degree.count(graph[last_node].edges[j].edge) ){ 
					other_node=graph[last_node].edges[j].edge; 
					is_uninteresting=false;
					break; }
				}

				if(is_uninteresting){continue;} // this is the middle of a lollipop => product of 2 lollipops and therefore not interesting

				// now last_node and other_node are the edge that connect the lollipop to the rest of the graph, this edge needs to be removed manually

				(*graph_clean[tid]) << graph[last_node].full_name << "\t" << species[graph[last_node].species_id] << "\t" << graph[other_node].full_name << "\t" << species[graph[other_node].species_id] << "\n";
			}
		}

		if(did_cut_lollipop){
			cur_cc.m_content_CC.erase(
				remove_if(
					cur_cc.m_content_CC.begin(), 
					cur_cc.m_content_CC.end(),
					[lollipop2nodes,node2lollipop,min_lollipop_length](const unsigned int & o) mutable -> bool { return node2lollipop.count(o) && lollipop2nodes [ node2lollipop[ o ] ].size()>=min_lollipop_length; }),
				cur_cc.m_content_CC.end()
			);
		}
		
		if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"  removeLollipopSubGraph_and_dispatch_allCores end size=" + to_string( cur_cc.size() ));
	}
	status.lollipop = true;

	status.use_all_cores=true;
	q->dispatch_allCores([cur_cc,q,status]{ partition_CC(cur_cc,q,status); });
}

void find_CCs_givenNodes(dynamic_queue *q, vector<unsigned int> todo_work ){

	string id_todo_work="";
	if(debug_level > 0 ) id_todo_work=getCCid(todo_work);

	map<unsigned int, bool> done;	// Keep track on what was done (for each node)
	bool allNodesAreDone = false;

	vector<ConnectedComponent> CC; // vector of all connected components found

	// find all nodes outside todo_work and remove them from the done vector !
	for (unsigned int i = 0 ; i < todo_work.size() ; i++) {
		unsigned int from = todo_work[i]; 
		done[ from ] = 0;
	}
	unsigned int number_edges=0; 
	for (unsigned int i = 0 ; i < todo_work.size() ; i++) {
		unsigned int from = todo_work[i]; 
		for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
			if(!done.count(graph[from].edges[j].edge)){
				done[ graph[from].edges[j].edge ]=1; // mark adjacent nodes that are NOT part of the input nodes explicitly as done, so they are ignored
			}else if(done[graph[from].edges[j].edge]==0){
				number_edges++;
			}
		}
	}
	if(number_edges==0){return;} // this can happen if the input nodes are disconnected 
	if(id_todo_work == "0" && debug_level>0){
		q->dispatch_log("[DEBUG:find_CCs_givenNodes] @0 : adj "+to_string(number_edges));
		for (unsigned int i = 0 ; i < todo_work.size() ; i++) {
			unsigned int from = todo_work[i]; 
			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
				if(done.count(graph[from].edges[j].edge) && done[graph[from].edges[j].edge]==0){
					q->dispatch_log("[DEBUG:find_CCs_givenNodes] @0 : "+to_string(from)+"-"+to_string(graph[from].edges[j].edge)+"-"+to_string(graph[from].edges[j].weight));
				}
			}
		}
	}

	if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:find_CCs_givenNodes] @" +id_todo_work+" size=" + to_string( todo_work.size() ) );

	while( true ){ // CC.size() < num_cpus / gather up to num_cpus connected components
		
		if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:find_CCs_givenNodes] @" +id_todo_work+" while" );
		allNodesAreDone = true;

		for (unsigned int i = 0 ; i < todo_work.size() ; i++) {

			// here it could lead to status where all cores are sleeping
			// ! therefore dont restrict here !

			unsigned int protein_id=todo_work[i];
			if ( graph[protein_id].is_done ){continue;}
			if ( done.count(protein_id) && done[protein_id] ){continue;}// We were here already

			if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:find_CCs_givenNodes] @" +id_todo_work+" start=" + to_string( protein_id ) );

			done[protein_id]=true; // mark this node
			ConnectedComponent cur_cc = BFS(&done,protein_id,0); // get the CC of the current node (protein_id) 

			// Do not report singles
			if (cur_cc.size() < param_minnodes) {
				for (unsigned int i = 0; i < cur_cc.size(); i++) { graph[cur_cc[i]].is_done=true; }
				continue;
			} // singletons are from no interest

			if(debug_level>0) q->dispatch_log ( getTime() + "[DEBUG] ConnectedComponent: @" + id_todo_work + "=>" + to_string( cur_cc.size() ) + "@" + ( getCCid(cur_cc.m_content_CC) ) );

			// Skip those that are too large (try heuristic)
			if (param_max_nodes > 0 && cur_cc.size() > param_max_nodes) {
				// reset done vector
				// for (unsigned int i = 0; i < cur_cc.size(); ++i){ done[cur_cc[i]]=false; }
				q->dispatch_log ( " [INFO]  Found a very large connected component that contains " + to_string(cur_cc.size()) + ">" +to_string( param_max_nodes )+ " (maxnodes) elements. For this component the fast but coarse heuristic removeLowQualityEdges will be used (involving the Smirnov-Grubb outlier test)." + ( debug_level>0 ? "@"+getCCid(todo_work) : "" ) );
				q->dispatch([cur_cc,q]{ removeLowQualityEdges(cur_cc,q,false); });
				continue;
			}

			if(param_core && cur_cc.species_num==0){ // species_num is later used for param_coreMaxProteinsPerSpecies
				if(cur_cc.size()<param_coreMinSpecies){allNodesAreDone=false;break;} // if there are less nodes than wanted species -> skip
				map<unsigned int,bool>cc_species;
				for (unsigned int i = 0; i < cur_cc.size(); ++i){ cc_species[graph[cur_cc[i]].species_id]=true; }
				cur_cc.species_num=cc_species.size();
				if(cur_cc.species_num<param_coreMinSpecies){allNodesAreDone=false;break;} // less species as wanted -> skip
			}
			
			cur_cc.calc_dsum();
			cur_cc.calc_density();
			if(cur_cc.density > 1){ cerr<< "[WARNING] : The input graph has duplicated edges, this lead to an invalid graph density of " +to_string( cur_cc.density ) + " (should be <1). Please clean the .blast-graph with 'proteinortho.pl --cleanblast --step=3 --project=...' or use the cleanupblastgraph tool in src/ to remove the duplicated edges." << "\n"; }
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG:find_CCs_givenNodes] Found connected component: " +to_string(cur_cc.size()) + " proteins (ID: " +to_string( protein_id ) +"), graph density="+to_string( cur_cc.density ) + ", sum of degrees="+to_string(cur_cc.d_sum ) + " @"+id_todo_work );

			q->dispatch([cur_cc,q]{ partition_status status; partition_CC(cur_cc,q,status); });
			
			allNodesAreDone=false;
		}
		if(allNodesAreDone)break; // no additional CC can be found -> done
	}
	if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:find_CCs_givenNodes] @" +id_todo_work+" done" );
}

void partition_CC(ConnectedComponent cur_cc, dynamic_queue *q, partition_status status){ //  bool do_lapack, unsigned int restarted
	size_t tid=std::hash<std::thread::id>{}(std::this_thread::get_id());

	if (debug_level > 0)
		q->dispatch_log ( "[DEBUG:partition_CC] start");

	string id_cur_cc="";
	if(debug_level>0) id_cur_cc = getCCid(cur_cc.m_content_CC);

	vector<float> x_hat; // the fiedler vector containing the split of the input nodes
	connectivity connectivity(0); // the connectivity value

	if( param_double == 2 && status.restarted==0 ){status.restarted=1;} // force double
	if( param_double == 0 && status.restarted>0  ){return;} // force float

	if(cur_cc.size()==0){return;} // sanity check

	if (param_con_threshold == -1) {
		print_group(cur_cc,connectivity,tid,false,q);
		return;
	}

	if( cur_cc.size()<=1 ){ 
		return;
	}else if( cur_cc.size() == 2 ){ 
		connectivity.value=1;
		print_group(cur_cc,connectivity,tid,false,q);
		return;
	}else if( cur_cc.size() == 3 && cur_cc.size() < param_max_nodes ){ 
		unsigned int num_edges=0;
		unsigned int a=cur_cc[0];
		unsigned int b=cur_cc[1];
		unsigned int c=cur_cc[2];

		for (unsigned int j = 0; j < graph[a].edges.size(); ++j){
			if(graph[a].edges[j].edge == b || graph[a].edges[j].edge == c)
				num_edges+=1;
		}
		for (unsigned int j = 0; j < graph[b].edges.size(); ++j){
			if(graph[b].edges[j].edge == a || graph[b].edges[j].edge == c)
				num_edges+=1;
		}
		for (unsigned int j = 0; j < graph[c].edges.size(); ++j){
			if(graph[c].edges[j].edge == a || graph[c].edges[j].edge == b)
				num_edges+=1;
		}

		if(num_edges>=6){ // full K3 (every edge is counted twice)
			connectivity.value=1;
		}else{ // one edge is missing, not perfect jet pretty good
			connectivity.value=0.33; // calculated 
		}

		if( param_core || connectivity.value > param_con_threshold ){
			print_group(cur_cc,connectivity,tid,false,q);
			return;
		}
	}
	
	if(cur_cc.density==-1){
		cur_cc.calc_dsum();
		cur_cc.calc_density();
	}

	if( !status.do_lapack ){
		// here start the power iteration job
		if (debug_level > 0)
			q->dispatch_log ( "[DEBUG:partition_CC] @"+id_cur_cc+" power size:" + to_string( cur_cc.size() ) + " d:" + to_string( cur_cc.density ) + " cut:" + to_string( (double)3.542144e-06*(double)cur_cc.size()-0.01701203 ));
		if(debug_level==10){
			q->dispatch_log ( getTime() + " [debug:10] extract cc" );
			ofstream OFS(("cur_cc_n"+to_string(cur_cc.size())+"_d"+to_string(cur_cc.density)+".ids").c_str());
			for (unsigned int i = 0; i < cur_cc.size(); ++i){
				OFS << graph[cur_cc[i]].full_name << "\n";
			}
			OFS.close();
		}
		if(status.restarted>0){
			vector<double> x_hat_dbl;
			connectivity = getConnectivity_double(&cur_cc.m_content_CC,false,&x_hat_dbl,status.use_all_cores,q);
			x_hat = std::vector<float>(x_hat_dbl.begin(), x_hat_dbl.end());
		}else
			connectivity = getConnectivity_float(&cur_cc.m_content_CC,false,&x_hat,status.use_all_cores,q);
	}else if(
		param_useLapack == 0 || ( // force the lapack algorithm with all cores
			status.do_lapack && // status is lapack (default), but the size of the CC is large (see below), therefore use power iteration with all cores!
			param_useLapack != 2 && // only allow dispatch_allCores if power iteration is allowed (-lapack 2 => only lapack, no power) 
			(
				cur_cc.size() >= 32768 || 
				( 
					( param_lapack_power_threshold_d> -1 && cur_cc.density < param_lapack_power_threshold_d ) || 
					( param_lapack_power_threshold_d==-1 && cur_cc.density < (double)3.542144e-06*(double)cur_cc.size()-0.01701203 ) 
				)
			)
		)
	){
		// now resubmit the CC using power and in the allCores queue
		if (debug_level > 0)
			q->dispatch_log ( "[DEBUG:partition_CC] @"+id_cur_cc+" dispatch all cores size:" + to_string( cur_cc.size() ) + " d:" + to_string( cur_cc.density ) + " cut:" + to_string((double)3.542144e-06*(double)cur_cc.size()-0.01701203));
		status.do_lapack=false;
		status.restarted=0;
		q->dispatch([cur_cc,q,status]{ removeLollipopSubGraph_and_dispatch_allCores(cur_cc,q,status); });
		//q->dispatch_allCores([cur_cc,q,status]{ partition_CC(cur_cc,q,status); });
		return;
	}else{
		// start the lapack iteration job
		if (debug_level > 0)
			q->dispatch_log ( "[DEBUG:partition_CC] @"+id_cur_cc+" lapack/power (single core application) size:" + to_string(cur_cc.size()) + " d:" + to_string( cur_cc.density ) );
		if(status.restarted>0){
			// if failed once -> retry with double precision instead of float
			vector<double> x_hat_dbl;
			connectivity = getConnectivity_double(&cur_cc.m_content_CC,true,&x_hat_dbl,status.use_all_cores,q);
			x_hat = std::vector<float>(x_hat_dbl.begin(), x_hat_dbl.end());
		}else
			connectivity = getConnectivity_float(&cur_cc.m_content_CC,true,&x_hat,status.use_all_cores,q);
	}

	if(param_core && cur_cc.species_num==0){
		map<unsigned int,bool>cc_species;
		for (unsigned int i = 0; i < cur_cc.size(); ++i){ cc_species[graph[cur_cc[i]].species_id]=true; }
		cur_cc.species_num=cc_species.size();
	}
	if(param_core && cur_cc.species_num < param_coreMinSpecies){return;}

	if (connectivity.param_min_species_is_true || param_con_threshold == 0 || (!param_core && abs(connectivity.value) > param_con_threshold)) {
		// cerr << " [DEBUG] done "<<connectivity<<"\n";
		if(param_more) cur_cc.x_hat=x_hat;
		if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:partition_CC] exit because : param_min_species_is_true || connectivity > param_con_threshold @"+id_cur_cc+" conn="+to_string(connectivity.value) );
		
		print_group(cur_cc,connectivity,tid,false,q);
		return;
	}
	// 5.17 new threshold option overwrites connectivity
	if ( param_min_species >=1 ) {
		if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:partition_CC]  Start the calculation of the average gene/species score @"+id_cur_cc+"" );
		float avg = calc_group(&cur_cc.m_content_CC);
		if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:partition_CC]   Found " + to_string( avg ) + " genes/species on average. User asked for at least " + to_string( param_min_species ) +" @"+id_cur_cc );
		if (avg <= param_min_species) {
			if (debug_level > 0) q->dispatch_log ( getTime() + " [DEBUG:partition_CC]   Group is going to be accepted despite connectivity " +to_string( connectivity.value ) + " @"+id_cur_cc);
			// just to be safe from infinit loops due to rounding
			if (connectivity.value == 0) connectivity.value = 0.001;
			// no splitting despite bad connectivity
			
			if(debug_level > 0) q->dispatch_log ( " [DEBUG:partition_CC] @"+id_cur_cc+" param_min_species done " + to_string(connectivity.value));

			if(param_more) cur_cc.x_hat=x_hat;
			print_group(cur_cc,connectivity,tid,false,q);
			return;
		}
	}

	// Store data about two groups (Zero cannot be assigned with certainty)
	ConnectedComponent groupA, groupB, groupZero;
	map<unsigned int,bool> groupB_set;
	map<unsigned int,bool> groupZero_set;

	if(param_sep_purity==-1){
		// automatic purity detection

		unsigned int n = cur_cc.size();
		map<unsigned int,unsigned int> mapping;
		for(unsigned int i = 0; i < (unsigned int)n; i++){ mapping[cur_cc[i]] = i; }
		
		/*
		 * Automatic prurity detetection (implemented v6.1.2)
		 * purity = threshold of nodes that are neither part of the bisection result of the current split (negative vs positive x_hat entries)
		 *
		 * purity is imporatant if there are multiple clusters that can be split or there is no good split
		 * identified nodes (in purity boundary) are put back to the queue stack (=groupZero)
		 * 
		 * maximal purity = 0.12 -> get all nodes with |x_hat(v)|<0.12
		 * identify all possible purity thresholds 
		 * each purity threshold is evaluated with the total sum of edge weights that are removed with this cut
		 * find most optimal purity cut-off
		 */

		float new_purity=0;
		float best_cut=0; // stores the current best cut (sum of all edges removed by the given purity), first take a purity of 0 (no purity) and set this value (split + and - compartments of x_hat)
		vector<float> x_hat_candidate_purity = x_hat;
		for (unsigned int i = 0; i < x_hat_candidate_purity.size(); i++) { x_hat_candidate_purity[i]=abs(x_hat_candidate_purity[i]); } // purity is a positive threshold
		sort(x_hat_candidate_purity.begin(),x_hat_candidate_purity.end()); // these values are used as purity tests. Later only subset is used of values not too similar to each other
		for (unsigned int i = 0; i < x_hat.size(); i++) {
			if(x_hat[i] < 0) {
				unsigned int from=cur_cc[i];
				for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					if(x_hat[mapping[to]] > 0) {
						best_cut += graph[from].edges[j].weight;							
					}
				}
			}
		}
		if(debug_level>0)	q->dispatch_log ( "[DEBUG] @"+id_cur_cc+" initial purity=0 best_cut=" + to_string( best_cut ) );
		float last_test_purity = -1; // omit purity test that are very close to previous values, to reduce runtime
		int num_purity_tests=0; // test all of the first 25 purity values anyway
		for(unsigned int pi = 0; pi < (unsigned int)n; pi++){
			if( x_hat_candidate_purity[ pi ] > 0.12 ){break;} // there is no further valid value (sorted values)
			float test_purity = x_hat_candidate_purity[ pi ];
			if( test_purity == 0 || 
					test_purity == last_test_purity || 
					(num_purity_tests>25 && test_purity-last_test_purity<0.01) ){continue;} // current value is very similar to last purity -> omit
			float cut_value = 0;
			num_purity_tests++;
			last_test_purity = test_purity;
			unsigned int num_nodes = 0; // the number of nodes within the purity bounds
			for (unsigned int i = 0; i < x_hat.size(); i++) {
				if(abs(x_hat[i]) <= test_purity) { // add all the edges between this node (below purity) and nodes above the purity threshold to the cut value
					num_nodes++;
					unsigned int from=cur_cc[i];
					for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
						unsigned int to = graph[from].edges[j].edge;
						if(!mapping.count(to)){continue;} // sanity check if the edge is in the current CC
						if(abs(x_hat[mapping[to]]) > test_purity) {
							cut_value+=graph[from].edges[j].weight;							
						}
					}
				}
			}
			if( cut_value>0 && (best_cut==-1 || best_cut>cut_value) && num_nodes>1){ // update optimum
				new_purity = test_purity;
				best_cut = cut_value;
				if(debug_level>0)	q->dispatch_log ( "[DEBUG] @"+id_cur_cc+" initial purity=" + to_string(new_purity) + " best_cut=" + to_string( best_cut ) );
			}
		}

		if(new_purity>0.12){new_purity=0;} // if there are no impure nodes -> disable
		unsigned int num_impure_nodes=0;
		for (unsigned int i = 0; i < x_hat.size(); i++) { if(abs(x_hat[i]) < new_purity) { num_impure_nodes++; } }
		if(debug_level>0)	q->dispatch_log ( "[DEBUG] @"+id_cur_cc+" num_impure_nodes=" + to_string( num_impure_nodes ) );
		if(num_impure_nodes<2 && new_purity > 0.001){new_purity=0; } // disable if there is only one node that is impure with a high purity threshold

		if(debug_level>0)	q->dispatch_log ( " [DEBUG] @"+id_cur_cc+" detected a purity of " + (new_purity==0 ? "<disabled>":to_string(new_purity)) );

		for (unsigned int i = 0; i < x_hat.size(); i++) {
			if(abs(x_hat[i]) <= new_purity) {
				groupZero.m_content_CC.push_back(cur_cc[i]);
				groupZero_set[cur_cc[i]] = true;
			}
			else if (x_hat[i] < 0) {
				groupA.m_content_CC.push_back(cur_cc[i]);
			}
			else { // x_hat[i] > 0
				groupB.m_content_CC.push_back(cur_cc[i]);
				groupB_set[cur_cc[i]] = true;
			}
		}

		if( (groupA.size() == 0 && groupB.size() == 0) ){

			q->dispatch_log( " [WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" All nodes are below the purity threshold. Continue without purity." );

			for (unsigned int i = 0; i < groupZero.size(); i++) {
				if (x_hat[ groupZero[i] ] < 0) {
					groupA.m_content_CC.push_back(groupZero[i]);
				}
				else { // x_hat[i] > 0
					groupB.m_content_CC.push_back(groupZero[i]);
				}
			}
		}
	}else{
		for (unsigned int i = 0; i < x_hat.size(); i++) {
			if(abs(x_hat[i]) < param_sep_purity) {
				groupZero.m_content_CC.push_back(cur_cc[i]);
				groupZero_set[cur_cc[i]] = true;
			}
			else if (x_hat[i] < 0) {
				groupA.m_content_CC.push_back(cur_cc[i]);
			}
			else { // x_hat[i] > 0
				groupB.m_content_CC.push_back(cur_cc[i]);
				groupB_set[cur_cc[i]] = true;
			}
		}
		if( (groupA.size() == 0 && groupB.size() == 0) ){
				
			q->dispatch_log( " [WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" All nodes are below the purity threshold. Continue purity/2." );

			for (unsigned int i = 0; i < x_hat.size(); i++) {
				if(abs(x_hat[i]) < param_sep_purity/2) {
					groupZero.m_content_CC.push_back(cur_cc[i]);
					groupZero_set[cur_cc[i]] = true;
				}
				else if (x_hat[i] < 0) {
					groupA.m_content_CC.push_back(cur_cc[i]);
				}
				else { // x_hat[i] > 0
					groupB.m_content_CC.push_back(cur_cc[i]);
					groupB_set[cur_cc[i]] = true;
				}
			}

			if( (groupA.size() == 0 && groupB.size() == 0) ){
					
				q->dispatch_log( " [WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" All nodes are below the purity threshold. Continue purity/10." );

				for (unsigned int i = 0; i < x_hat.size(); i++) {
					if(abs(x_hat[i]) < param_sep_purity/10) {
						groupZero.m_content_CC.push_back(cur_cc[i]);
						groupZero_set[cur_cc[i]] = true;
					}
					else if (x_hat[i] < 0) {
						groupA.m_content_CC.push_back(cur_cc[i]);
					}
					else { // x_hat[i] > 0
						groupB.m_content_CC.push_back(cur_cc[i]);
						groupB_set[cur_cc[i]] = true;
					}
				}

				if( (groupA.size() == 0 && groupB.size() == 0) ){
						
					q->dispatch_log( " [WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" All nodes are below the purity threshold. Continue purity/100." );

					for (unsigned int i = 0; i < x_hat.size(); i++) {
						if(abs(x_hat[i]) < param_sep_purity/100) {
							groupZero.m_content_CC.push_back(cur_cc[i]);
							groupZero_set[cur_cc[i]] = true;
						}
						else if (x_hat[i] < 0) {
							groupA.m_content_CC.push_back(cur_cc[i]);
						}
						else { // x_hat[i] > 0
							groupB.m_content_CC.push_back(cur_cc[i]);
							groupB_set[cur_cc[i]] = true;
						}
					}

					if( (groupA.size() == 0 && groupB.size() == 0) ){
							
						q->dispatch_log( " [WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" All nodes are below the purity threshold. Continue without purity." );

						for (unsigned int i = 0; i < groupZero.size(); i++) {
							if (x_hat[ groupZero[i] ] < 0) {
								groupA.m_content_CC.push_back(groupZero[i]);
							}
							else { // x_hat[i] > 0
								groupB.m_content_CC.push_back(groupZero[i]);
							}
						}
					}
				}
			}
		}
	}

	string id=""; string idA=""; string idB=""; string idZ="";
	if(debug_level==15){ 
		/*
		 * This is a debug function, used for printing out all CC (use this with -debug 15)
		 */
		unsigned int n = cur_cc.size();
		id=id_cur_cc;
		idA=getCCid(groupA.m_content_CC);
		idB=getCCid(groupB.m_content_CC);
		idZ=getCCid(groupZero.m_content_CC);
		map<unsigned int,unsigned int> mapping;
		for (unsigned int i = 0; i < (unsigned int)n; i++) {mapping[cur_cc[i]] = i;}
		{
			ofstream OFS(("cluster_n"+to_string(n)+"_id"+id+".edgelist").c_str());
			OFS << "source\ttarget\tweight"<<endl;

			map<pair<unsigned int,unsigned int>,unsigned int> knownedges;
			for(unsigned int i = 0 ; i < (unsigned int)n ; i++){
				unsigned int from = cur_cc[i]; 
				unsigned int sum = 0;
				if(!mapping.count(from)){continue;}
				for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					pair<unsigned int,unsigned int> key_cur_edge;
					if(from < to) key_cur_edge = make_pair(from,to);
					else key_cur_edge = make_pair(to,from);
					if(!knownedges.count(key_cur_edge)){
						knownedges[key_cur_edge] = true;
						OFS << graph[from].full_name << "\t" << graph[to].full_name << "\t" << graph[from].edges[j].weight << endl;
					}
				}
			}
			OFS.close();
		}
		{
			ofstream OFS(("cluster_n"+to_string(n)+"_id"+id+"_ci0_conn"+to_string(connectivity.value)+".nodeweight").c_str());
			OFS << "id\teigenvector"<<endl;
			for(unsigned int i = 0 ; i < (unsigned int)n ; i++){
				OFS << graph[cur_cc[i]].full_name << "\t" << x_hat[i] << endl;
			}
			OFS.close();
		}
		{
			ofstream OFS(("cluster_n"+to_string(n)+"_id"+id+"_ci0_conn"+to_string(connectivity.value)+".split").c_str());
			OFS << "id\tsplit"<<endl;
			for (unsigned int i = 0; i < groupZero.size(); i++) { OFS << graph[groupZero[i]].full_name << "\t0" << endl; }
			for (unsigned int i = 0; i < groupA.size(); i++) { OFS << graph[groupA[i]].full_name << "\t1" << endl; }
			for (unsigned int i = 0; i < groupB.size(); i++) { OFS << graph[groupB[i]].full_name << "\t-1" << endl; }
			OFS.close();
		}
	}

	if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG:partition_CC] "+(debug_level>0 ? "@"+id_cur_cc:"")+" splitting CC with " + to_string(cur_cc.size()) + " nodes " + (debug_level==15 ? "@"+id+"" : "") + " into (" + to_string(groupA.size()) + (debug_level==15 ? " @"+idA+"" : "") + "," + to_string(groupB.size()) + (debug_level==15 ? " @"+idB+"" : "") + "," + to_string(groupZero.size()) + (debug_level==15 ? " @"+idZ+"" : "") + ") sized groups!");

	// Catch error in laplacien calcs
	// all nodes are in one partition -> restart with an increased restart level
	if ( (groupA.size() == 0 && groupB.size() == 0) || 
		 ( (groupA.size() == 0 || groupB.size() == 0) && groupZero.size() == 0) ){

		if(status.restarted==0){
			if(debug_level>0)
				q->dispatch_log ( getTime() + " [WARNING] "+(debug_level>0 ? "@"+id_cur_cc:"")+" connected component with " + to_string(cur_cc.size())+" nodes and density of "+ to_string(cur_cc.density)+": the fiedler vector associated to the algebraic connectivity of "+ to_string(connectivity.value)+" would result in invalid split groups ("+to_string(groupA.size())+","+to_string(groupB.size())+","+to_string(groupZero.size())+"). I will retry with double precision, if this persists an error will be thrown.");
			status.do_lapack=true;
			status.restarted++;
			status.use_all_cores=false;
			q->dispatch([cur_cc,q,status]{ partition_CC(cur_cc,q,status); });
		}else if(status.restarted==1){
			if(debug_level>0)
				q->dispatch_log ( getTime() + " [WARNING] "+(debug_level>0 ? "@"+id_cur_cc:"")+" connected component with "+ to_string(cur_cc.size())+" nodes and density of "+ to_string(cur_cc.density)+": the fiedler vector associated to the algebraic connectivity of "+ to_string(connectivity.value)+" would result in invalid split groups ("+to_string(groupA.size())+","+to_string(groupB.size())+","+to_string(groupZero.size())+"). I will retry with double precision using the power iteration, if this persists an error will be thrown.");
			status.do_lapack=false;
			status.restarted++;
			q->dispatch([cur_cc,q,status]{ removeLollipopSubGraph_and_dispatch_allCores(cur_cc,q,status); });
			//q->dispatch_allCores([cur_cc,q,status]{ partition_CC(cur_cc,q,status); });
		}else{

			q->dispatch_log ( getTime() + " [ERROR] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" connected component with "+ to_string(cur_cc.size())+" nodes and density of "+ to_string(cur_cc.density)+": the fiedler vector associated to the algebraic connectivity of "+ to_string(connectivity.value)+" would result in invalid split groups ("+to_string(groupA.size())+","+to_string(groupB.size())+","+to_string(groupZero.size())+"). Failed even with double precision, now using removeLowQualityEdges !");

			q->dispatch([cur_cc,q]{ removeLowQualityEdges(cur_cc,q,false); });
		}
		
	}else{

		// for the -core algorithm, test if the number of species now decreased by too much or keep on splitting
		if(param_core && param_coreMinAlgCon < connectivity.value){
			map<unsigned int,bool>cc_speciesA;
			for (unsigned int i = 0; i < groupA.size(); ++i){ cc_speciesA[graph[groupA[i]].species_id]=true; }
			groupA.species_num=cc_speciesA.size();
			map<unsigned int,bool>cc_speciesB;
			for (unsigned int i = 0; i < groupB.size(); ++i){ cc_speciesB[graph[groupB[i]].species_id]=true; }
			groupB.species_num=cc_speciesB.size();
			map<unsigned int, bool> done; // for zero additionally test if this group is connected -> if not

			if(groupA.species_num!=cur_cc.species_num && // neither A, B or Zero meet the -core criteria -> test zero and if zero also do not hold -> print the original group cur_cc before the split
					groupB.species_num!=cur_cc.species_num && 
					param_coreMaxProteinsPerSpecies*cur_cc.species_num > cur_cc.size()){

				map<unsigned int,bool>cc_speciesZero;
				for (unsigned int i = 0; i < groupZero.size(); ++i){ cc_speciesZero[graph[groupZero[i]].species_id]=true; }
				groupZero.species_num=cc_speciesZero.size();

				if( groupZero.species_num!=cur_cc.species_num ){ // the original CC is still fine -> print
					if(param_more) cur_cc.x_hat=x_hat;
					print_group(cur_cc,connectivity,tid,false,q);
					return; // done with this group, no need to split into A and B by the definition of -core
				}
			}
		}

		// 6.2.3
		// if the split is execptionally bad, try the removeLowQualityEdges instead (only one iteration with the Smirnov-Grubbs test cutoff)
		// test if the split disects the connected component
		// if yes, send those new CC back to the queue
		// if no, use the current "bad split"
		unsigned int n=groupA.size()+groupB.size()+groupZero.size();
		if( n > 5000 && 
			( groupA.size()+groupB.size() < n/100 || groupA.size()+groupZero.size() < n/100 || groupB.size()+groupZero.size() < n/100 ) ){

			if(debug_level>0)
				q->dispatch_log ( "I will retry with the removeLowQualityEdges heuristic (using only the Smirnov-Grubb test cutoff).");

			if(debug_level>0)
				q->dispatch_log ( getTime() + " [WARNING] "+(debug_level>0 ? "@"+id_cur_cc:"")+" connected component with " + to_string(cur_cc.size())+" nodes and density of "+ to_string(cur_cc.density)+": has a bad split groups ("+to_string(groupA.size())+","+to_string(groupB.size())+","+to_string(groupZero.size())+"). I will retry with the removeLowQualityEdges heuristic (using only the Smirnov-Grubb test cutoff).");

			map<unsigned int, bool> done;
			for (unsigned int i = 0; i < cur_cc.size(); i++) {
				done[cur_cc[i]]=0;
			}

			// find the lowest 10% of edge values (min+p*(max-min))
			float min_w = -1; 
			float max_w = -1;
			vector<double> data;
			unsigned int original_number_nodes=cur_cc.size();
			for (unsigned int i = 0; i < cur_cc.size(); i++) {
				unsigned int from=cur_cc[i];
				for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
					unsigned int to = graph[from].edges[j].edge;
					if(!done.count(to)){continue;}
					unsigned int weight = graph[from].edges[j].weight;
					data.push_back( weight<=0 ? 0 : log((double)weight) ); // log weights s.t. they are more normally distributed ...
					if(min_w == -1 || weight < min_w){min_w=weight;} 
					if(min_w == -1 || weight > max_w){max_w=weight;}
				}
			}
			float cut_off = (float)exp(lowerSmirnovGrubbsTest_binary_search_cutoff(data)); // transform back

			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+"  removeLowQualityEdges:ini size=" + to_string(cur_cc.size()) + " cut_off(lowerSmirnovGrubbsTest_binary_search_cutoff)=" + to_string(cut_off) + " fallback="+to_string(min_w+0.1*(max_w-min_w))+ " min_w=" +to_string(min_w) + " max_w=" + to_string(max_w));

			// fallback use the lowest 10%
			if(cut_off<=0){ cut_off=min_w+cut_off*(max_w-min_w); }

			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+"  removeLowQualityEdges:ini2 size=" + to_string(cur_cc.size()) + " cut_off=" + to_string(cut_off) + " min_w=" +to_string(min_w) + " max_w=" + to_string(max_w));

			for (unsigned int i = 0 ; i < cur_cc.size() ; i++) {
				unsigned int from = cur_cc[i]; 
				for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
					if(!done.count(graph[from].edges[j].edge))
						done[ graph[from].edges[j].edge ]=1;
				}
			}

			vector<ConnectedComponent> output;

			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges size=" + to_string(cur_cc.size()) + " while ");
			
			for (unsigned int id = 0 ; id < cur_cc.size() ; id++) {
			
				unsigned int protein_id = cur_cc[id];

				if ( (done.count(protein_id) && done[protein_id]) || graph[protein_id].is_done){continue;}// We were here already

				if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc + "   removeLowQualityEdges BFS start" );

				done[protein_id]=true; // mark this node
				ConnectedComponent sub_work_cc = BFS(&done,protein_id,cut_off); // get the CC of the current node (protein_id) 

				if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+"  removeLowQualityEdges intial_cc.size=" + to_string(cur_cc.size()) + " find_CCs:start at " + to_string( protein_id ) + " found CC of sub_work_cc.size=" + to_string( sub_work_cc.size() ));

				if ( sub_work_cc.size() == cur_cc.size() ) {
					if(debug_level>0 && param_max_nodes>0 && sub_work_cc.size() > param_max_nodes) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+" removeLowQualityEdges Found a very large connected component that contains " + to_string(sub_work_cc.size()) + ">" + to_string(param_max_nodes) +" (maxnodes) elements. For this component the fast but coarse heuristic removeLowQualityEdges will be used (involving the Smirnov-Grubb outlier test).");
					break;
				}

				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+ " :removeLowQualityEdges calc_dsum,calc_density ");
				sub_work_cc.calc_dsum();
				sub_work_cc.calc_density();
				if(sub_work_cc.density > 1){ cerr << "[WARNING] : The input graph has duplicated edges, this lead to an invalid graph density of " << sub_work_cc.density << " (should be <1). Please clean the .blast-graph with 'proteinortho.pl --cleanblast --step=3 --project=...' or use the cleanupblastgraph tool in src/ to remove the duplicated edges." << "\n"; }
				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] @" +id_cur_cc+"->@" +getCCid(sub_work_cc.m_content_CC)+ " :removeLowQualityEdges Found connected component: " +to_string(sub_work_cc.size())+ " proteins (ID: "+ to_string(protein_id) + "), graph density=" + to_string( sub_work_cc.density )+ ", sum of degrees=" + to_string(sub_work_cc.d_sum)+ " ini from " + graph[protein_id].full_name + " of size=" + to_string(sub_work_cc.size()));

				if(sub_work_cc.size()>1){ // dont report singles
					output.push_back(sub_work_cc);
				}
			}

			if(debug_level>0) q->dispatch_log( " [DEBUG] @" +id_cur_cc+" removeLowQualityEdges.just_smirnov_grubbs did fail and the CC is resubmitted to the stack. original size="+to_string(cur_cc.size()));

			bool did_find_a_better_split=false;
			// now test if the new CCs are a better split
			// true = if there is a CC that is large but not too large
			for (unsigned int i = 0; i < output.size(); ++i){
				if( output[i].size() > cur_cc.size()/100 &&
					output[i].size() < cur_cc.size()*99/100 ){
					did_find_a_better_split=true;
					break;
				}
			}

			if(debug_level>0)
				q->dispatch_log ( "did_find_a_better_split="+to_string(did_find_a_better_split)+" output="+to_string(output.size()));

			if(!did_find_a_better_split){
				for (unsigned int i = 0; i < output.size(); ++i){
					ConnectedComponent sub_work_cc = output[i];
					q->dispatch([sub_work_cc,q]{ partition_status status; partition_CC(sub_work_cc,q,status); });
				}
				return; // new splitted components are allready back in queue
			} // else it did not work, so just work with the current bad split, nothing to improve on with this approach
		}

		// now add the edges between A and B (and Zero) to the output rmgraph
		// (the rmgraph contains all edges that are removed by the clustering)

		if(!graph_clean.count(tid))
			graph_clean[tid]=make_shared<ofstream>((param_rmgraph+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(tid)).c_str());

		if(debug_level>0) q->dispatch_log (  "[INFO] "+(debug_level>0 ? "@"+id_cur_cc:"")+" removing between A and B+Zero" );
		// remove edges between A and (B or Zero)
		for(unsigned int i = 0 ; i < groupA.size() ; i++){
			protein node_i = graph[groupA[i]];
			for(unsigned int j = 0 ; j < graph[groupA[i]].edges.size() ; j++){
				protein node_j = graph[graph[groupA[i]].edges[j].edge];
				if(groupB_set.count(graph[groupA[i]].edges[j].edge) || 
					groupZero_set.count(graph[groupA[i]].edges[j].edge) ){
					(*graph_clean[tid]) << node_i.full_name << "\t" << species[node_i.species_id] << "\t" << node_j.full_name << "\t" << species[node_j.species_id] << "\n";
				}
			}
		}

		if(debug_level>0) q->dispatch_log ( "[INFO] "+(debug_level>0 ? "@"+id_cur_cc:"")+" removing between B and Zero");
		// remove edges between B and Zero
		for(unsigned int i = 0 ; i < groupB.size() ; i++){
			protein node_i = graph[groupB[i]];
			for(unsigned int j = 0 ; j < graph[groupB[i]].edges.size() ; j++){
				protein node_j = graph[graph[groupB[i]].edges[j].edge];
				if(groupZero_set.count(graph[groupB[i]].edges[j].edge)){
					(*graph_clean[tid]) << node_i.full_name << "\t" << species[node_i.species_id] << "\t" << node_j.full_name << "\t" << species[node_j.species_id] << "\n";
				}
			}
		}

		// finally add the new clusters back to the stack of CCs
		// for -core only add those clusters which meet the criteria of number of species

		if(debug_level>0) q->dispatch_log ( "[INFO] "+(debug_level>0 ? "@"+id_cur_cc:"")+" dispatching groupA="+to_string(groupA.size())+", groupB="+to_string(groupB.size())+", groupZero="+to_string(groupZero.size()));

		if(param_core){
			if((groupA.species_num >= param_coreMinSpecies) && groupA.size()>1)
				q->dispatch([groupA,q]{ find_CCs_givenNodes(q,groupA.m_content_CC); });
			if((groupB.species_num >= param_coreMinSpecies) && groupB.size()>1)
				q->dispatch([groupB,q]{ find_CCs_givenNodes(q,groupB.m_content_CC); });
			if((groupZero.species_num >= param_coreMinSpecies) && groupZero.size()>1) // only if zero is connected 
				q->dispatch([groupZero,q]{ find_CCs_givenNodes(q,groupZero.m_content_CC); });
		}else{
			if(groupA.size()>1)
				q->dispatch([groupA,q]{ find_CCs_givenNodes(q,groupA.m_content_CC); });
			if(groupB.size()>1)
				q->dispatch([groupB,q]{ find_CCs_givenNodes(q,groupB.m_content_CC); });
			if(groupZero.size()>1)
				q->dispatch([groupZero,q]{ find_CCs_givenNodes(q,groupZero.m_content_CC); });
		}
	}
}

void find_CCs(dynamic_queue *q){

	map<unsigned int, bool> done;	// Keep track on what was done (for each node)
	bool allNodesAreDone = false;

	vector<ConnectedComponent> CC; // vector of all connected components found

	while( true ){ // CC.size() < num_cpus / gather up to num_cpus connected components

		if(debug_level>0) q->dispatch_log ( " [DEBUG:find_CCs] while" );

		allNodesAreDone = true;

		for (unsigned int protein_id = 0 ; protein_id < graph.size() ; protein_id++) {
			
			// dont overflow the system with CC, wait until less than 1k CC are present
			if(num_cpus > 1 && q->pending()>2000){
				if(debug_level>0) q->dispatch_log ( "[DEBUG:find_CCs] WARNING queue overload, sleeping the find_CC loop" ); 
				while(q->pending()>1000){ sleep(1); }
			}

			if ( graph[protein_id].is_done ){continue;}
			if ( done.count(protein_id) && done[protein_id] ){continue;}// We were here already

			done[protein_id]=true; // mark this node
			ConnectedComponent cur_cc = BFS(&done,protein_id,0); // get the CC of the current node (protein_id) 

			// Do not report singles
			if (cur_cc.size() < param_minnodes) {
				for (unsigned int i = 0; i < cur_cc.size(); i++) { graph[cur_cc[i]].is_done=true; }
				continue;
			} // singletons are from no interest / or lower than the defined cutoff

			if(debug_level>0) q->dispatch_log (  getTime() + " [DEBUG:find_CCs] q="+to_string(q->pending())+" find_CCs:start at "+ to_string( protein_id )+ "<" +to_string( graph.size()));

			// for -core, skip CC with less than param_coreMinSpecies species
			if( param_core && cur_cc.species_num==0 ){
				if(cur_cc.size()<param_coreMinSpecies){allNodesAreDone=false;break;} // if there are less nodes than wanted species -> skip
				map<unsigned int,bool>cc_species;
				for (unsigned int i = 0; i < cur_cc.size(); ++i){ cc_species[graph[cur_cc[i]].species_id]=true; }
				cur_cc.species_num=cc_species.size();
				if(cc_species.size()<param_coreMinSpecies){allNodesAreDone=false;break;} // less species as wanted -> skip
			}

			// Skip those that are too large (try heuristic)
			if (param_max_nodes > 0 && cur_cc.size() > param_max_nodes) {
				q->dispatch_log (  " [INFO] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" Found a very large connected component that contains " + to_string( cur_cc.size() ) + ">" + to_string( param_max_nodes ) + " (maxnodes) elements. For this component the fast but coarse heuristic removeLowQualityEdges will be used (involving the Smirnov-Grubb outlier test)." );
				q->dispatch([cur_cc,q]{ removeLowQualityEdges(cur_cc,q,true); });
				continue;
			}

			if(job_n>0){ unsigned int id = getCCid_uint(cur_cc.m_content_CC[0]); if( id % job_n != job_i - 1 ){ continue; } }

			if (debug_level > 0) q->dispatch_log (  getTime() +" [DEBUG:find_CCs] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+" Found connected component: " +to_string( cur_cc.size() )+ " proteins (ID: "+ to_string( protein_id )+ "), ini from " + graph[protein_id].full_name);
			cur_cc.calc_dsum();
			cur_cc.calc_density();
			if(cur_cc.density > 1){ cerr << ( "[WARNING] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+": The input graph has duplicated edges, this lead to an invalid graph density of " +to_string(cur_cc.density ) + " (should be <1). Please clean the .blast-graph with 'proteinortho.pl --cleanblast --step=3 --project=...' or use the cleanupblastgraph tool in src/ to remove the duplicated edges.") << endl; }
			if (debug_level > 0) q->dispatch_log (  getTime() + " [DEBUG:find_CCs] "+(debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC):"")+"did calc dsum and density: " +to_string( cur_cc.size() ) + " proteins (ID: " +to_string( protein_id ) + "), graph density="+to_string( cur_cc.density ) + ", sum of degrees="+to_string( cur_cc.d_sum) + " ini from " + graph[protein_id].full_name);

			q->dispatch([cur_cc,q]{ partition_status status; partition_CC(cur_cc,q,status); });
			allNodesAreDone=false;
			//break;
		}
		if(allNodesAreDone)break; // no additional CC can be found -> done
	}
}

///////////////////////////////////////////////////////////
// Major partioning algorithm
///////////////////////////////////////////////////////////

void partition_graph() {
	dynamic_queue q( num_cpus );

	q.dispatch([&q]{ find_CCs(&q); });
	q.start();

	if(debug_level>0) cerr << getTime() << " [DEBUG] done with ini dynamic_queue" << endl;

	q.waitTilDone();

	if(debug_level>0) cerr << getTime() << " [DEBUG] quit dynamic_queue" << endl;

	return;
}

///////////////////////////////////////////////////////////
// File parser
///////////////////////////////////////////////////////////
void process_line(string *line,string *file_a,string *file_b, unsigned int *file_a_id,unsigned int *file_b_id,bool *take_prots_as_species,float *avg_bitscore_median){
	vector<string> fields;
	tokenize(*line, fields, "\t");
	bool isHeader = line->size()>0 && fields[0].substr(0, 1) == "#";
	// Header line
	if ( (fields.size() == 2 || fields.size() == 6) && isHeader) {
		*file_a = fields[0].substr(2, fields[0].size()-2);
		*file_b = fields[1];

		if (*file_a == "file_a" && *file_b == "file_b") return;	// Header line
		if (*file_a == "a" && *file_b == "b") return;	// Header line

		if(fields.size() == 6){ // either the median scores are directly after the species header OR see <else if>
			*avg_bitscore_median=(string2float(fields[3])+string2float(fields[5]))/2;
			if(*avg_bitscore_median<1){*avg_bitscore_median=-1;}
		}

		// Map species a
		if (species2id.find(*file_a) == species2id.end())	{
				species.push_back(*file_a);
				species2id[*file_a] = species_counter++;
		}
		// Map species b
		if (species2id.find(*file_b) == species2id.end())	{
				species.push_back(*file_b);
				species2id[*file_b] = species_counter++;
		}

		*file_a_id = species2id[*file_a];
		*file_b_id = species2id[*file_b];
	}else if ( (fields.size() == 4) && isHeader && fields[0].substr(0, 7) != "# Score") { // OR the median scores are in an additional line below the species header

		*avg_bitscore_median=(string2float(fields[1])+string2float(fields[3]))/2;
		if(*avg_bitscore_median<1){*avg_bitscore_median=-1;}

	}
	// Data line
	else if ((fields.size() == 6 || fields.size() == 8) && !isHeader) {
		// a b e1 b1 e2 b2 score

		// for blast-graphs without any header information -> the species field is empty (file_a,file_b)
		// just take the protein ids as species names and continue
		if(*file_a == "" && *file_b == ""){
			cerr << "[WARNING] incomplete blast-graph format: The header lines are missing. I will just assume each protein is a species entry too. (minspecies is disabled)" << endl;
			param_min_species=0;
			*take_prots_as_species=1;
			species2id["input"] = species_counter++;
			species.push_back("input");
		}
		if(*take_prots_as_species){
			*file_a=fields[0];
			*file_b=fields[1];
		}

		// 5.16 deal with duplicated IDs by adding file ID to protein ID
		string ida = fields[0];
		string idb = fields[1];
		if(!*take_prots_as_species){ fields[0] += " "; fields[0] += to_string(*file_a_id); }
		if(!*take_prots_as_species){ fields[1] += " "; fields[1] += to_string(*file_b_id); }

		// 5.16 do not point to yourself
		if (!fields[0].compare(fields[1])) {return;}

		// A new protein
		if (protein2id.find(fields[0]) == protein2id.end())	{
			protein a;
			a.full_name	= ida;
			a.species_id = *take_prots_as_species ? 0 : *file_a_id;
			protein2id[fields[0]] = protein_counter++;

			if( graph.size() >= 1073741824-1 ){
				cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
			}
			graph.push_back(a);
		}
		if (protein2id.find(fields[1]) == protein2id.end())	{
			protein b;
			b.full_name	= idb;
			b.species_id = *take_prots_as_species ? 0 : *file_b_id;
			protein2id[fields[1]] = protein_counter++;

			if( graph.size() >= 1073741824-1 ){
				cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
			}
			graph.push_back(b);
		}

		// Bitscores 
		// check range (in float)

		float bit_a,bit_b; 
		if(*avg_bitscore_median<0){
			bit_a = string2float(fields[3]);//255 is exactly in the middle of ushort range, such that the score can be 255 fold upregulated or down regulated.
			bit_b = string2float(fields[5]);
		}else{	
			bit_a = 255.0* (string2float(fields[3])/(*avg_bitscore_median));//255 is exactly in the middle of ushort range, such that the score can be 255 fold upregulated or down regulated.
			bit_b = 255.0* (string2float(fields[5])/(*avg_bitscore_median));
		}

		if(bit_a<1){bit_a=1;}
		if(bit_b<1){bit_b=1;}

		if(bit_a>USHRT_MAX){
			cerr << " [WARNING] unsigned short overflow " << bit_a <<  ">USHRT_MAX (bitscore of "<< ida<< " adj. to "<< idb<< ") using "<< USHRT_MAX<< " (USHRT_MAX) instead." << "\n";
			bit_a=(float)USHRT_MAX;
		}
		if(bit_b>USHRT_MAX){
			cerr << " [WARNING] unsigned short overflow " << bit_b <<  ">USHRT_MAX (bitscore of "<< idb<< " adj. to "<< ida<< ") using "<< USHRT_MAX<< " (USHRT_MAX) instead." << "\n";
			bit_b=(float)USHRT_MAX;
		}

		// assign
		unsigned short bitscore_avg = (bit_a + bit_b)/2;
		if(bitscore_avg<1){bitscore_avg=1;}

		// Add link to graph (reciprocal)					
		unsigned int a_id = protein2id[fields[0]];
		unsigned int b_id = protein2id[fields[1]];

		// if(!param_useWeights){bitscore_avg=1;} // weights are still needed for the maxnodes heuristic

		// 5.17, add weight
		wedge w;
		w.edge=b_id;
		w.weight=bitscore_avg;
		graph[a_id].edges.push_back(w);
		w.edge=a_id;
		w.weight=bitscore_avg;
		graph[b_id].edges.push_back(w);
		edges++;
	}
}

void parse_file(string file) {
	string file_a = "";	unsigned int file_a_id = 0;
	string file_b = "";	unsigned int file_b_id = 0;
	bool take_prots_as_species=0;
	float avg_bitscore_median=-1; //for normalization the bitscore is devided by the median bitscore

	if(file == "-"){
		if (param_verbose) cerr << "Reading STDIN\n";
		for (std::string line; std::getline(std::cin, line);) {
	        process_line(&line,&file_a,&file_b,&file_a_id,&file_b_id,&take_prots_as_species,&avg_bitscore_median);
	    }
	}else{
		if (param_verbose) cerr << "Reading " << file << "\n";
		string line;
		ifstream graph_file(file.c_str());
		if (graph_file.is_open()) {
			// For each line
			while (!graph_file.eof()) {
				getline(graph_file, line);
				process_line(&line,&file_a,&file_b,&file_a_id,&file_b_id,&take_prots_as_species,&avg_bitscore_median);
			}
			graph_file.close();
			if (debug_level) cerr << "done " << file << "\n";
		}
		else {
			cerr << string("Could not open file " + file).c_str() << "\n";throw;
		}
	}
	
	if(species_counter==0){species.push_back("0");species_counter++;}
}

void process_line_abc(string *line, map<string, bool> * seen, bool *take_prots_as_species){
	vector<string> fields;
	unsigned int file_a_id=0;
	unsigned int file_b_id=0;
	tokenize(*line, fields, "\t");
	// Header line
	if(fields.size() == 2){ fields.push_back("1"); }
	if ((fields.size() == 3) && fields[0].substr(0, 1) != "#") {
		// a b e1 b1 e2 b2 score

		// 5.16 deal with duplicated IDs by adding file ID to protein ID
		string ida = fields[0];
		string idb = fields[1];
		if((*seen).count(ida+"---"+idb)){
			cerr << "[WARNING] I found the edge ("<<ida<<","<<idb<<") multiple times, I will only use the first occurence"<< endl;
			return;
		}
		(*seen)[ida+"---"+idb]=1;
		(*seen)[idb+"---"+ida]=1;

		vector<string> fields_ida;
		tokenize(ida, fields_ida, "--");
		vector<string> fields_idb;
		tokenize(idb, fields_idb, "--");

		if((fields_ida.size()!=2 || fields_idb.size()!=2) && *take_prots_as_species==0){
			param_min_species=0;
			*take_prots_as_species=1;
			species2id["input"] = species_counter++;
			species.push_back("input");
		}else{
			// Map species a
			if (species2id.find(fields_ida[1]) == species2id.end())	{
				species.push_back(fields_ida[1]);
				species2id[fields_ida[1]] = species_counter++;
			}
			// Map species b
			if (species2id.find(fields_idb[1]) == species2id.end())	{
				species.push_back(fields_idb[1]);
				species2id[fields_idb[1]] = species_counter++;
			}

			file_a_id = species2id[fields_ida[1]];
			file_b_id = species2id[fields_idb[1]];

			ida=fields_ida[0];
			idb=fields_idb[0];
		}

		// 5.16 do not point to yourself
		if (!fields[0].compare(fields[1])) {return;}

		// A new protein
		if (protein2id.find(fields[0]) == protein2id.end())	{
			protein a;
			a.full_name	= ida;
			// a.species_id = 0;
			a.species_id = *take_prots_as_species ? 0 : file_a_id;
			protein2id[fields[0]] = protein_counter++;

			if( graph.size() >= 1073741824-1 ){
				cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
			}
			graph.push_back(a);
		}
		if (protein2id.find(fields[1]) == protein2id.end())	{
			protein b;
			b.full_name	= idb;
			//b.species_id	= 0;
			b.species_id = *take_prots_as_species ? 0 : file_b_id;
			protein2id[fields[1]] = protein_counter++;

			if( graph.size() >= 1073741824-1 ){
				cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
			}
			graph.push_back(b);
		}

		// Bitscores 
		float bit_a = string2float(fields[2]);

		if(bit_a<1){bit_a=1;}

		if(bit_a>USHRT_MAX){
			cerr << " [WARNING] unsigned short overflow " << bit_a <<  ">USHRT_MAX (bitscore of "<< ida<< " adj. to "<< idb<< ") using "<< USHRT_MAX<< " (USHRT_MAX) instead." << "\n";
			bit_a=(float)USHRT_MAX;
		}

		// Add link to graph (reciprocal)					
		unsigned int a_id = protein2id[fields[0]];
		unsigned int b_id = protein2id[fields[1]];

		// 5.17, add weight
		wedge w;
		w.edge=b_id;
		w.weight=bit_a;
		graph[a_id].edges.push_back(w);
		w.edge=a_id;
		w.weight=bit_a;
		graph[b_id].edges.push_back(w);
		edges++;
	}
}

void parse_abc_file(string file) {

	map<string, bool> seen;	
	bool take_prots_as_species=0;

	if(file == "-"){
		if (param_verbose) cerr << "Reading STDIN\n";
		for (std::string line; std::getline(std::cin, line);) {
	        process_line_abc(&line,&seen,&take_prots_as_species);
	    }
	}else{
		if (param_verbose) cerr << "Reading " << file << "\n";
		string line;
		ifstream graph_file(file.c_str());
		if (graph_file.is_open()) {
			while (!graph_file.eof()) {
				getline(graph_file, line);
				process_line_abc(&line,&seen,&take_prots_as_species);
			}
			graph_file.close();
		}
		else {
			cerr << string("Could not open file " + file).c_str() << "\n";throw;
		}
	}

	if(species_counter==0){species.push_back("0");species_counter++;}
}


///////////////////////////////////////////////////////////
// Output
///////////////////////////////////////////////////////////
// Sort
void sort_species(void) {

	reorder_table.reserve(species_counter);
	vector<string> species_sorted (species_counter);
	copy(species.begin(), species.end(), species_sorted.begin());
	sort(species_sorted.begin(), species_sorted.end());
	// find new locations (not efficent but list is small)	
	for (unsigned int i = 0; i < species_counter; i++) {
		for (unsigned int j = 0; j < species_counter; j++) {
			if (species[i] == species_sorted[j]) {reorder_table[j] = i; continue;}
		}
	}
}

// Progress stats
void stats(unsigned int lapack,unsigned int power,bool active_status) {
	// this is not part of the dynamic_queue and not protected by the lock as clashes do not matter
	// this way it is slightly more performant
	if (!param_verbose) return;
	if (last_stat_lapack != lapack || last_stat_power != power || last_stat_act != active_status) {
		last_stat_lapack = lapack; 
		last_stat_power = power;
		last_stat_act = active_status;
		if(debug_level==0) cerr << '\r';
		cerr << "Clustering: working on (" << lapack << (lapack==1 ? "@BFS/lapack" : "@lapack")<<(active_status?"*":"")<<" + " << power << "@power"<<(!active_status?"*":"")<<") connected component"<< (lapack+power>1?"s":"") <<"          " << std::flush;
		if(lapack==0 && power==0){ cerr << '\r'<<"Clustering: done                                                       \n"; }
		if(debug_level>0) cerr << endl;
	}
}

// Header with species names
void print_header() {
	cout << "# Species\tGenes\tAlg.-Conn.";
	for (unsigned int i = 0; i < species_counter; i++) {
		cout << "\t" << species[reorder_table[i]];
	}
	cout << "\n";
}

struct protein_degree{
	inline bool operator() (const pair<string,int> & p1, const pair<string,int>& p2){
		return (p1.second > p2.second);
	}
};

// Group formatting
void print_group(ConnectedComponent& cur_cc, connectivity connectivity, size_t tid, bool failed, dynamic_queue *q) {

	string id_cur_cc="";
	if(debug_level>0) id_cur_cc=getCCid(cur_cc.m_content_CC);

	for (unsigned int i = 0; i < cur_cc.size(); i++) { graph[cur_cc[i]].is_done=true; }

	if(cur_cc.size()<2){return;}

	map<unsigned int, bool> done;	// Keep track on what was done (for each node)
	for (unsigned int i = 0 ; i < cur_cc.size() ; i++) {
		unsigned int from = cur_cc[i]; 
		done[ from ] = 0;
	}
	for (unsigned int i = 0 ; i < cur_cc.size() ; i++) {
		unsigned int from = cur_cc[i]; 
		for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
			if(!done.count(graph[from].edges[j].edge))
				done[ graph[from].edges[j].edge ]=1;
		}
	}

	if(param_prepare>0){
		unsigned int id = (getCCid_uint(cur_cc.m_content_CC[0])) % param_prepare;
		ofstream out;
		out.open((param_rmgraph+"_job"+to_string(id)+"_tid"+to_string(tid)+".blast-graph").c_str(), std::ios_base::app);
		for (unsigned int i = 0 ; i < cur_cc.size() ; i++) {
			unsigned int from = cur_cc[i]; 
			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
				unsigned int to=graph[from].edges[j].edge;
				if(from < to || !done.count(to) || done[to]){ continue; }
				out << graph[from].full_name << "--" << species[graph[from].species_id] << "\t" << graph[to].full_name << "--" << species[graph[to].species_id] << "\t" << graph[from].edges[j].weight << endl;
			}
		}
		out.close();
		return;
	}

	if(!proteinorthotmp_clean.count(tid)){
		proteinorthotmp_clean[tid]=make_shared<ofstream>((param_rmgraph+"_proteinortho_tmp_"+(job_n > 0 ? "_"+to_string(job_i)+"_"+to_string(job_n)+"_":"")+to_string(tid)).c_str());
	}

	vector<vector<pair<string,int> > > line(species_counter);	// Output vector

	unsigned int species_number = 0;
	// For each protein in group 	 	
	for (unsigned int i = 0; i < cur_cc.size(); i++) {
		unsigned int current_protein = cur_cc[i];
		unsigned int current_species = graph[current_protein].species_id;
		if (line[current_species].size() == 0) 
			species_number++;
		line[current_species].push_back(make_pair( graph[current_protein].full_name , graph[current_protein].edges.size() ));
	}


	(*proteinorthotmp_clean[tid]) << species_number << "\t" << cur_cc.size() << "\t" << (debug_level>0 ? "@"+getCCid(cur_cc.m_content_CC)+":":"") << setprecision (3) << connectivity.value << (failed ? "*": "");
	
	// List group data
	for (unsigned int i = 0; i < species_counter; i++) {
		
		string return_line="";

		// sort line 
		if (line[reorder_table[i]].size() > 0) {

			sort( line[reorder_table[i]].begin(), line[reorder_table[i]].end(), protein_degree() );

			return_line = line[reorder_table[i]][0].first;

			for (unsigned int k = 1; k < line[reorder_table[i]].size(); k++) {
				return_line.append(","+line[reorder_table[i]][k].first);
			}
		}
		if(return_line == "")
			return_line = "*";

		// output
		(*proteinorthotmp_clean[tid]) << "\t" << return_line;
	}

	if(debug_level>0) q->dispatch_log(getTime() + " [DEBUG] print_group @" + id_cur_cc + " ERROR=" + to_string((int)cur_cc.size() - (int)cur_cc.size()) );

	if(param_more){
		// just output the connected component with connectivity

		(*proteinorthotmp_clean[tid]) << "\t";

		(*proteinorthotmp_clean[tid]) << "id="<< getCCid(cur_cc.m_content_CC) << ";n="<< cur_cc.size() << ";species=" << species_number <<";density=" << cur_cc.density<< ";connectivity=" << connectivity.value;
		
		(*proteinorthotmp_clean[tid]) << ";nodes=";
		for (unsigned int i = 0; i < cur_cc.size(); ++i){
			if(i>0){ (*proteinorthotmp_clean[tid]) << ","; }
			(*proteinorthotmp_clean[tid]) << cur_cc[i];
		}
		(*proteinorthotmp_clean[tid]) << ";nodelabels=";
		for (unsigned int i = 0; i < cur_cc.size(); ++i){
			if(i>0){ (*proteinorthotmp_clean[tid]) << ","; }
			(*proteinorthotmp_clean[tid]) << graph[cur_cc[i]].full_name;
		}
		(*proteinorthotmp_clean[tid]) << ";species=";
		for (unsigned int i = 0; i < cur_cc.size(); ++i){
			if(i>0){ (*proteinorthotmp_clean[tid]) << ","; }
			(*proteinorthotmp_clean[tid]) << species[graph[cur_cc[i]].species_id];
		}

		(*proteinorthotmp_clean[tid]) << ";adjacencylist=";
		for (unsigned int i = 0; i < cur_cc.size(); i++) {
			unsigned int from=cur_cc[i];
			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){
				if(i>0 || j>0){ (*proteinorthotmp_clean[tid]) << ","; }
				unsigned int to = graph[from].edges[j].edge;
				unsigned int weight = graph[from].edges[j].weight;
				(*proteinorthotmp_clean[tid]) << from << "-" << to << ":" << weight;
			}
		}

		(*proteinorthotmp_clean[tid]) << ";fiedler=";
		for (unsigned int i = 0; i < cur_cc.size(); ++i){
			if(i>0){ (*proteinorthotmp_clean[tid]) << ","; }
			(*proteinorthotmp_clean[tid]) << cur_cc.x_hat[i];
		}
	
	}

	(*proteinorthotmp_clean[tid]) << "\n";
}

// Calculate number of species formatting
float calc_group(vector<unsigned int>* nodes) {

	map<unsigned int, bool> speciesids;

	for (unsigned int i = 0; i < nodes->size(); i++) {
		unsigned int current_protein = (*nodes)[i];
		speciesids[graph[current_protein].species_id]=1;
	}

	return speciesids.size()==0 ? 99999 : ((float)nodes->size())/((float)speciesids.size()); // 99999 such that if the species information is missing, then the criterion always fails and the splits are only made based on the alg. connectivity
}

///////////////////////////////////////////////////////////
// Misc functions
///////////////////////////////////////////////////////////
// Convert string to float
float string2float(string str) {
	istringstream buffer(str);
	float value;
	buffer >> value;
	return value;
}
float string2double(string str) {
	istringstream buffer(str);
	double value;
	buffer >> value;
	return value;
}

// Split a string at a certain delim
void tokenize(const string& str, vector<string>& tokens, const string& delimiters = "\t") {

	// Skip delimiters at beginning.
	string::size_type lastPos = str.find_first_not_of(delimiters, 0);
	// Find first "non-delimiter".
	string::size_type pos = str.find_first_of(delimiters, lastPos);

	while (string::npos != pos || string::npos != lastPos) {
		// Found a token, add it to the vector.
		tokens.push_back(str.substr(lastPos, pos - lastPos));
		// Skip delimiters.  Note the "not_of"
		lastPos = str.find_first_not_of(delimiters, pos);
		// Find next "non-delimiter"
		pos = str.find_first_of(delimiters, lastPos);
	}
}

///////////////////////////////////////////////////////////
// Algebraic connectivity functions
///////////////////////////////////////////////////////////
// Return maximum degree of given protein_ids -- openMP A ML
unsigned int max_of_diag(vector<unsigned int>& nodes, vector<unsigned int>& diag, bool use_all_cores) {

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] max_of_diag start" << endl;

	unsigned int max = 0;

	bool useOpenMpFlag = (use_all_cores && nodes.size() > param_minOpenmp);

	#pragma omp parallel for num_threads(num_cpus) reduction(max: max) if (useOpenMpFlag)
	for (unsigned int i = 0; i < nodes.size(); i++) {
		if (diag[i] > max) max = diag[i] ;
	}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] max_of_diag end" << endl;
	return max;
}

// Generate random vector x of size size
template<class T> vector<T> generate_random_vector(const unsigned int size) {

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] generate_random_vector start" << endl;

	vector<T> x(size);

	x[0] = (T)(rand() % 999+1)/1000;	// 0 to 1
	for (unsigned int i = 1; i < size; i++) {
		x[i] = (T)(rand() % 999+1)/1000;	// 0 to 1
		if (x[i] == x[i-1]) x[i] /= 3;		// Check: at least one value must be different from the others but still within 0 and 1
	}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] generate_random_vector end" << endl;

	return x;
}

// determine new X, Formula (1) -- openMP B ML
template<class T> vector<T> get_new_x(vector<T>*x, vector<unsigned int>& nodes, map<unsigned int,unsigned int> &mapping, bool isWeighted, bool use_all_cores) {
	
	if(debug_level==9) cerr << getTime() << " [DEBUG:9] get_new_x start" << endl; 

	vector<T> x_new(x->size(),0);

	bool useOpenMpFlag = (use_all_cores && nodes.size() > param_minOpenmp);
	if(debug_level==9) cerr << getTime() << " [DEBUG:9] get_new_x done ini " << useOpenMpFlag << endl;

	if(isWeighted){

		#pragma omp parallel for num_threads(num_cpus) if (useOpenMpFlag)
		for (unsigned int i = 0; i < nodes.size(); i++) {
			// go through adjacency list of node 
			for (unsigned int j = 0; j < graph[nodes[i]].edges.size(); j++) {
				// y points to z, so take entry z from x
				unsigned int abs_target = graph[nodes[i]].edges[j].edge;
				if(!mapping.count(abs_target)){continue;}
				unsigned int rel_target = mapping[abs_target];

				x_new[i] += (*x)[rel_target]*(T)graph[nodes[i]].edges[j].weight;
			}
		}

	}else{

		#pragma omp parallel for num_threads(num_cpus) if (useOpenMpFlag)
		for (unsigned int i = 0; i < nodes.size(); i++) {

			// go through adjacency list of node 
			for (unsigned int j = 0; j < graph[nodes[i]].edges.size(); j++) {
				// y points to z, so take entry z from x
				unsigned int abs_target = graph[nodes[i]].edges[j].edge;
				if(!mapping.count(abs_target)){continue;}
				unsigned int rel_target = mapping[abs_target];

				x_new[i] += (*x)[rel_target];
			}
		}
	}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] get_new_x end" << endl;

	return x_new;
}

// Make vector x orthogonal to 1, Formula (2) -- openMP A ML
template<class T> vector<T> makeOrthogonal(vector<T> x, bool use_all_cores) {

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] makeOrthogonal start" << endl;

	T sum = 0;

	bool useOpenMpFlag = (use_all_cores && x.size() > param_minOpenmp);

	#pragma omp parallel for num_threads(num_cpus) reduction(+: sum) if (useOpenMpFlag)
	for (unsigned int i = 0; i < x.size(); i++) {sum += x[i];}

	T average = sum/x.size();

	#pragma omp parallel for num_threads(num_cpus) if (useOpenMpFlag)
	for (unsigned int i = 0; i < x.size(); i++) {x[i] -= average;}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] makeOrthogonal end" << endl;
	return x;
}

// Normalize vector x, Formula (4) -- openMP A ML
template<class T> void normalize(vector<T> *x, T *length, bool use_all_cores) {

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] normalize start" << endl;
	T sum = 0;

	bool useOpenMpFlag = (use_all_cores && x->size() > param_minOpenmp);

	#pragma omp parallel for num_threads(num_cpus) reduction(+: sum) if (useOpenMpFlag)
	for (unsigned int i = 0; i < x->size(); i++) {sum += (*x)[i]*(*x)[i];}

	*length = (T)sqrt(sum);
	if (*length == 0) {*length = 0.000000001;}// ATTENTION not 0!

	#pragma omp parallel for num_threads(num_cpus) if (useOpenMpFlag)
	for (unsigned int i = 0; i < x->size(); i++) {(*x)[i] /= *length;}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] normalize end" << endl;
}

// Qx, Formula (5) -- openMP A ML
template<class T> vector<T> getY(T max_degree, vector<T> x_hat, vector<T>* x_new, vector<unsigned int>& nodes, vector<unsigned int>& diag, bool use_all_cores){

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] getY start" << endl;

	bool useOpenMpFlag = (use_all_cores && nodes.size() > param_minOpenmp);

	#pragma omp parallel for num_threads(num_cpus) if (useOpenMpFlag)
	for (unsigned int i = 0; i < x_hat.size(); i++) {
		x_hat[i] *= ((T)2*max_degree - (T)diag[i]);
		x_hat[i] += (*x_new)[i];
	}

	if(debug_level==9) cerr << getTime() << " [DEBUG:9] getY end" << endl;

	return x_hat;
}

connectivity getConnectivity_double(vector<unsigned int> *nodes, bool allow_useLapack, vector<double> *x_hat,bool use_all_cores,dynamic_queue *q) {
	// allow_useLapack = is submitted as a function call that should use lapack (based on size and graph density) but can be overwritten by -lapack 0 (use power always)

	string id_nodes="";
	if(debug_level>0) id_nodes=getCCid(*nodes);

	bool useWeights = ( param_useWeights==2 || param_useWeights==1 && nodes->size() <= param_max_nodes_weight ); //param_useWeights = user input whether weights should be used or not. useWeights = the true value, that is true if param_useWeights is true and the maximum number of nodes are not exeeded for the weighted algorithm (param_max_nodes_weight)

	unsigned int n=nodes->size();

	if( n>1073741824 ){
		cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
	}

	double maxWeight=-1;
	map<unsigned int,unsigned int> mapping;
	for (unsigned int i = 0; i < (unsigned int)n; i++) {mapping[(*nodes)[i]] = i;}

	connectivity connectivity(0);
	*x_hat = vector<double>(n);

	if( ( n < 32768 && allow_useLapack && param_useLapack == 1 ) || param_useLapack == 2 ){
		if (debug_level > 0) q->dispatch_log(getTime() + " [DEBUG]@double using LAPACK @" +id_nodes );
		if(debug_level>0) q->dispatch_status( 1 );

		// maximal number of nodes 32768 (2^15 = SHRT_MAX) -> laplace matrix 2^15*2^15=2^30 (max vector size) entries 
		// max vector size = std::vector<int> myvector; cout << myvector.max_size() << "\n"; -> 2^30
		// used ram in MB of lapack = (unsigned int)n*(unsigned int)n*sizeof(double))/1e+6 

		double * laplacian = (double*)calloc( (unsigned int)n*(unsigned int)n,sizeof(double) );

		bool fill_laplacian_return=1; // return value of the fill algorithm -> true : all fine, false : ERROR
		// fill laplacian
		for(unsigned int i = 0 ; i < (unsigned int)n ; i++){

			unsigned int from = (*nodes)[i]; 
			if(!mapping.count(from) || graph[from].is_done){continue;}
			unsigned int sum = 0;

			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){

				unsigned int to = graph[from].edges[j].edge;

				if(!mapping.count(to) || graph[to].is_done){continue;}
				unsigned int vector_idx = mapping[from] + mapping[to]*n; //transform the 2-d coordinate (i,j) of the nxn matrix to 1-d vector coordinate i+j*n of the 2n vector

				if( vector_idx >= (unsigned int)n*(unsigned int)n){
					fill_laplacian_return = false;
					break;
				}

				if( useWeights
					){
					double w = graph[from].edges[j].weight;
					sum+=w;
					laplacian[vector_idx]=-w;
					if(maxWeight<w)maxWeight=w;
				}else{
					sum++;
					laplacian[vector_idx]=-1.0;
				}
			}

			laplacian[mapping[from]+mapping[from]*n]=sum;
		}
		if(!fill_laplacian_return){
			cerr <<"CRITICAL ERROR : fill_laplacian : out of range" << "\n";
			throw;
		}

		// local variables:
			int il, iu, m = 1, lda = n, ldz = n, info, lwork, liwork, iwkopt;
			double vl, vu;
			double wkopt;
			double* work;
			int* iwork;
			int isuppz[(unsigned int)(2*m)];
			char Vchar='V', Ichar='I', Uchar='U'; // Ichar = for specific range of eigenvalues/vectors
			double eigenvalues[(unsigned int)n]; // need only 1 eigenvalue
			double * eigenvectors = (double*)malloc( (unsigned int)ldz*(unsigned int)m*sizeof(double) ); 
			il = 2; //that is the second one (il=1 -> the first one, il=2 the second one)
			iu = 2; 
			double eps=param_epsilon;

		// Determine optimal workspace 
			lwork = -1;
			liwork = -1;
			int n_int=(int)n;
			
			dsyevr_( &Vchar, &Ichar, &Uchar, &n_int, laplacian, &lda, &vl, &vu, &il, &iu, &eps, &m, eigenvalues, eigenvectors, &ldz, isuppz, &wkopt, &lwork, &iwkopt, &liwork, &info );
						
			lwork = (int)wkopt;
			work = (double*)malloc( lwork*sizeof(double) );
			liwork = iwkopt;
			iwork = (int*)malloc( liwork*sizeof(int) );

			// Solve eigenproblem ...
			
			dsyevr_( &Vchar, &Ichar,&Uchar, &n_int, laplacian, &lda, &vl, &vu, &il, &iu,&eps, &m, eigenvalues, eigenvectors, &ldz, isuppz, work, &lwork, iwork, &liwork , &info );
					
			// Check for errors in convergence
			if( info > 0 ) {
				cerr << " [ERROR] The algorithm (d|s)syevr failed to compute eigenvalues. Continue now with the slow standard approach (power iteration)." << "\n";
			throw;
			// goto standardComputationOfAlgCon;
		}

		// deallocate, malloc=>free
		free(laplacian);
		free(work);
		free(iwork);

		// calculate normalized algebraic connectivity and fill x_hat

		// eigenvalues[0] contains the second smalles eigenvalue as specified with il and ul above
		if(useWeights){
			connectivity.value = abs( eigenvalues[0]/(maxWeight*(double)n) );
		}else{
			connectivity.value = abs( eigenvalues[0]/((double)n) );
		}
		for(unsigned int i = 0 ; i < (unsigned int)n ; i++)
			(*x_hat)[i]=-eigenvectors[i];

		// deallocate
		free(eigenvectors);	

	}else{
		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]@double using POWER @" +id_nodes );
		if(debug_level>0) q->dispatch_status( 0 );

		if(debug_level==9) q->dispatch_log( getTime() + " [DEBUG:9] start" );

		if(param_useWeights && !useWeights){
			q->dispatch_log( " [INFO] The maximum number of nodes for the weighted algorithm is exeeded. Continue now with the faster unweighted algorithm." );
		}

		if(n<10){useWeights=true;} // for very small graphs always use the weighted algorithm 
		// otherwise the algorithm can go into a core dump, e.g. use the QfO2020_04 diamond blast-graph and set -lapack 0 -double 0 -weighted 0 and there are sometimes throws "Segmentation fault" (around 1 in 10 calls)
		// needs to be investigated further but this fixes this issue for now
		// this is not impactful as power iteration only is applied to graphs with such small n for the benchmarkings

		//diagonal matrix diag : d(u,u)=number of adjacency nodes=deg(u)
		vector<unsigned int> diag(n);

		if(useWeights){
			for (unsigned int i = 0; i < (unsigned int)n; i++) {
				diag[i]=0;
				unsigned int from = (*nodes)[i]; 
				if(!mapping.count(from)){continue;}

				for (unsigned int j = 0; j < graph[(*nodes)[i]].edges.size(); j++) {

					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					diag[i] += graph[(*nodes)[i]].edges[j].weight;
					if(maxWeight<graph[(*nodes)[i]].edges[j].weight)maxWeight=graph[(*nodes)[i]].edges[j].weight;
				}
			}
		}else{
			for (unsigned int i = 0; i < (unsigned int)n; i++) {
				diag[i]=0;
				unsigned int from = (*nodes)[i]; 
				if(!mapping.count(from)){continue;}
				for (unsigned int j = 0; j < graph[(*nodes)[i]].edges.size(); j++) {
					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					diag[i] += 1;
				}
			}
		}

		if(debug_level==9) cerr << getTime() << " [DEBUG:9] ini done" << endl;

		// Get max degree / sum of weights of nodes
		unsigned int max_d = max_of_diag((*nodes),diag,use_all_cores);	

		// Init randomized variables. 
		vector<double> x = generate_random_vector<double>(n);		

		// Orthogonalize + normalize vector + get initial lenght
		double current_length = 0;
		double last_length;

		(*x_hat) = makeOrthogonal<double>(x,use_all_cores);
		normalize(x_hat, &last_length, use_all_cores);

		// Repeat until difference < param_epsilon
		unsigned int iter = 0;	// catch huge clustering issues by keeping track here

		while(++iter < param_max_iter) { 

			if(debug_level==9) cerr << getTime() << " [DEBUG:9] iter:" << iter << endl;

			last_length = current_length;

			// Get a new x
			x = get_new_x<double>(x_hat, (*nodes), mapping, useWeights, use_all_cores);

			// Get y
			vector<double> y = getY<double>(max_d,*x_hat,&x,(*nodes),diag,use_all_cores);

			// Orthogonalize
			(*x_hat) = makeOrthogonal<double>(y,use_all_cores);

			// Get length (lambda) & normalize vector
			normalize(x_hat, &current_length, use_all_cores);

			if ( abs(current_length-last_length) < param_epsilon && iter >= min_iter ) break;	// prevent convergence by chance, converge to param_epsilon
		}
		if(debug_level==9) q->dispatch_log( getTime() + " [DEBUG:9] post while" );

		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   " + to_string(iter) + " / " + to_string( param_max_iter ) + " iterations required (error is " + to_string(abs(current_length-last_length)) + ") @" +id_nodes );

		#ifdef DEBUG
			total_number_of_iterations_convergence+=iter;
		#endif

		if(useWeights){
			connectivity.value = abs( (-current_length+(double)2*max_d)/(maxWeight*(double)n) );
		}else{
			connectivity.value = abs( (-current_length+(double)2*max_d)/((double)n) );
		}

		// 5.17 catch hardly-converging groups
		if (iter >= param_max_iter) {
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Connectivity score of connected component with " + to_string(n) + " elements did not converge perfectly in time." );
		}
	}

	if (debug_level > 1){
		string log= getTime() + " [DEBUG]   Connectivity score @" +id_nodes +":" + to_string( connectivity.value );
		if ( (debug_level > 1 && (unsigned int)n<100 ) || debug_level > 2 ){
			log += " eigenvector: (";
			for(unsigned int i = 0 ; i < (unsigned int)n ; i++)
				log += to_string((*x_hat)[i]) + ","; 
			log += ")";
		}
		q->dispatch_log(log);
	}else if(debug_level>0){
		q->dispatch_log(getTime() +" [DEBUG]   Connectivity score @" +id_nodes+":" + to_string( connectivity.value ));
	}

	// Split groups if connectivity is too low, remove tree like structures that might have arosen
	if (param_con_threshold > 0 && abs(connectivity.value) < param_con_threshold) {
		
		// 5.17 new threshold option overwrites connectivity
		if (param_min_species >=1 ) {
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG:getConnectivity_double]  Start the calculation of the average gene/species score @" +id_nodes );
			double avg = calc_group(nodes);
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Found " + to_string(avg) + " genes/species on average. User asked for at least " + to_string(param_min_species) +" @" +id_nodes );
			if (avg <= param_min_species) {
				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Group is going to be accepted despite connectivity " + to_string(connectivity.value) +" @" +id_nodes );
				// just to be safe from infinit loops due to rounding
				if (connectivity.value == 0) connectivity.value = 0.001;
				// no splitting despite bad connectivity
				connectivity.param_min_species_is_true=1;
				return connectivity;			
			}
		}
	}
	
	return connectivity;
}

// Yeah, getConnectivity_float is just getConnectivity_double but with float precision, 
// everything is the same and need to be update in parallel.
// This is bad but I did not manage to make one function for 
// both because (using template) of the fortran interface
connectivity getConnectivity_float(vector<unsigned int> *nodes, bool allow_useLapack, vector<float> *x_hat,bool use_all_cores,dynamic_queue *q){
	// allow_useLapack = is submitted as a function call that should use lapack (based on size and graph density) but can be overwritten by -lapack 0 (use power always)

	string id_nodes="";
	if(debug_level>0) id_nodes=getCCid(*nodes);

	if (debug_level > 0) q->dispatch_log( getTime() + " getConnectivity_float @" +id_nodes );

	bool useWeights = ( param_useWeights==2 || param_useWeights==1 && nodes->size() <= param_max_nodes_weight ); //param_useWeights = user input whether weights should be used or not. useWeights = the true value, that is true if param_useWeights is true and the maximum number of nodes are not exeeded for the weighted algorithm (param_max_nodes_weight)

	unsigned int n=nodes->size();

	if( n>1073741824 ){
		cerr << string("[CRITICAL ERROR]   Overflow: number of nodes overflow the maximum number for vector allocation (2^30=1073741824).").c_str() << "\n";throw;
	}

	float maxWeight=-1;
	map<unsigned int,unsigned int> mapping;
	for (unsigned int i = 0; i < (unsigned int)n; i++) {mapping[(*nodes)[i]] = i;}

	connectivity connectivity(0);
	*x_hat = vector<float>(n);

	if( ( n < 32768 && allow_useLapack && param_useLapack == 1 ) || param_useLapack == 2 ){

		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]@float using LAPACK @" +id_nodes );
		if(debug_level>0) q->dispatch_status( 1 );

		// maximal number of nodes 32768 (2^15 = SHRT_MAX) -> laplace matrix 2^15*2^15=2^30 (max vector size) entries 
		// max vector size = std::vector<int> myvector; cout << myvector.max_size() << "\n"; -> 2^30
		// used ram in MB of lapack = (unsigned int)n*(unsigned int)n*sizeof(float))/1e+6 

		float * laplacian = (float*)calloc( (unsigned int)n*(unsigned int)n,sizeof(float) );

		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] post alloc @" +id_nodes );

		bool fill_laplacian_return=1; // return value of the fill algorithm -> true : all fine, false : ERROR
		// fill laplacian
		for(unsigned int i = 0 ; i < (unsigned int)n ; i++){

			if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] i="+to_string(i)+" @" +id_nodes );

			unsigned int from = (*nodes)[i]; 

			if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] from="+to_string(from)+" @" +id_nodes );

			if(!mapping.count(from) || graph[from].is_done){continue;}
			unsigned int sum = 0;

			for(unsigned int j = 0 ; j < graph[from].edges.size() ; j++){

				if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] i="+to_string(i)+" j="+to_string(j)+" @" +id_nodes );

				unsigned int to = graph[from].edges[j].edge;

				if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] to="+to_string(to)+" @" +id_nodes );

				if(!mapping.count(to) || graph[to].is_done){continue;}
				unsigned int vector_idx = mapping[from] + mapping[to]*n; //transform the 2-d coordinate (i,j) of the nxn matrix to 1-d vector coordinate i+j*n of the 2n vector

				if( vector_idx >= (unsigned int)n*(unsigned int)n){
					fill_laplacian_return = false;
					break;
				}				

				if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] vector_idx="+to_string(vector_idx)+" @" +id_nodes );

				if( useWeights
					){
					if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] pre-w @" +id_nodes );

					float w = graph[from].edges[j].weight;
					sum+=w;

					if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] pre-set @" +id_nodes );

					laplacian[vector_idx]=-w;
					if(maxWeight<w)maxWeight=w;
				}else{
					sum++;
					laplacian[vector_idx]=-1.0;
				}
			}
			if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG] pre-sum set @" +id_nodes );
			if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG]  sum="+to_string(sum)+" @" +id_nodes );
			if (debug_level > 2) q->dispatch_log( getTime() + " [DEBUG]  at="+to_string(mapping[from]+mapping[from]*n)+" n2="+to_string((unsigned int)n*(unsigned int)n)+" @" +id_nodes );

			laplacian[mapping[from]+mapping[from]*n]=sum;
		}
		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG] post fill @" +id_nodes );
		if(!fill_laplacian_return){
			cerr <<"CRITICAL ERROR : fill_laplacian : out of range" << "\n";
			throw;
		}

		// local variables:
			int il, iu, m = 1, lda = n, ldz = n, info, lwork, liwork, iwkopt;
			float vl, vu;
			float wkopt;
			float* work;
			int* iwork;
			int isuppz[(unsigned int)(2*m)];
			char Vchar='V', Ichar='I', Uchar='U'; // Ichar = for specific range of eigenvalues/vectors
			float eigenvalues[(unsigned int)n]; // need only 1 eigenvalue
			il = 2; //that is the second one (il=1 -> the first one, il=2 the second one)
			iu = 2; 
			//if(debug_level==15){ iu = 5; } // generate more vectors for debug 15
			float * eigenvectors = (float*)malloc( (unsigned int)ldz*(unsigned int)m*(iu-il+1)*sizeof(float) ); 
			float eps=param_epsilon;

			// Determine optimal workspace 
			lwork = -1;
			liwork = -1;
			int n_int=(int)n;
			
			ssyevr_( &Vchar, &Ichar, &Uchar, &n_int, laplacian, &lda, &vl, &vu, &il, &iu, &eps, &m, eigenvalues, eigenvectors, &ldz, isuppz, &wkopt, &lwork, &iwkopt, &liwork, &info  );
						
			lwork = (int)wkopt;
			work = (float*)malloc( lwork*sizeof(float) );
			liwork = iwkopt;
			iwork = (int*)malloc( liwork*sizeof(int) );

			// Solve eigenproblem ...
			ssyevr_( &Vchar, &Ichar,&Uchar, &n_int, laplacian, &lda, &vl, &vu, &il, &iu,&eps, &m, eigenvalues, eigenvectors, &ldz, isuppz, work, &lwork, iwork, &liwork , &info  );
						
			// Check for errors in convergence
			if( info > 0 ) {
				cerr << " [ERROR] The algorithm (d|s)syevr failed to compute eigenvalues. Continue now with the slow standard approach (power iteration)." << "\n";
			throw;
			// goto standardComputationOfAlgCon;
		}

		// deallocate
		free(laplacian);
		free(work);
		free(iwork);

		// calculate normalized algebraic connectivity and fill x_hat

		// eigenvalues[0] contains the second smalles eigenvalue as specified with il and ul above
		if(useWeights){
			connectivity.value = abs( (double)eigenvalues[0]/(maxWeight*(double)n) );
		}else{
			connectivity.value = abs( (double)eigenvalues[0]/((double)n) );
		}
		for(unsigned int i = 0 ; i < (unsigned int)n ; i++)
			(*x_hat)[i]=-eigenvectors[i]; 

		free(eigenvectors);	
		if(debug_level>0) q->dispatch_log(getTime() +" [DEBUG]   Connectivity score @" +id_nodes+":" + to_string( connectivity.value )+ " = (" + to_string(eigenvalues[0])+")/("+to_string(maxWeight)+"*"+to_string(n)+")" );

	}else{

		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]@float using POWER @" +id_nodes );
		if(debug_level>0) q->dispatch_status( 0 );

		if(param_useWeights && !useWeights){
			q->dispatch_log( " [INFO] The maximum number of nodes for the weighted algorithm is exeeded. Continue now with the faster unweighted algorithm." );
		}

		if(n<10){useWeights=true;} // for very small graphs always use the weighted algorithm 
		// otherwise the algorithm can go into a core dump, e.g. use the QfO2020_04 diamond blast-graph and set -lapack 0 -double 0 -weighted 0 and there are sometimes throws "Segmentation fault" (around 1 in 10 calls)
		// needs to be investigated further but this fixes this issue for now
		// this is not impactful as power iteration only is applied to graphs with such small n for the benchmarkings

		//diagonal matrix diag : d(u,u)=number of adjacency nodes=deg(u)
		vector<unsigned int> diag(n);

		if(useWeights){
			for (unsigned int i = 0; i < (unsigned int)n; i++) {
				diag[i]=0;
				unsigned int from = (*nodes)[i]; 
				if(!mapping.count(from)){continue;}

				for (unsigned int j = 0; j < graph[(*nodes)[i]].edges.size(); j++) {

					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					diag[i] += graph[(*nodes)[i]].edges[j].weight;
					if(maxWeight<graph[(*nodes)[i]].edges[j].weight)maxWeight=graph[(*nodes)[i]].edges[j].weight;
				}
			}
		}else{
			for (unsigned int i = 0; i < (unsigned int)n; i++) {
				diag[i]=0;
				unsigned int from = (*nodes)[i]; 
				if(!mapping.count(from)){continue;}
				for (unsigned int j = 0; j < graph[(*nodes)[i]].edges.size(); j++) {
					unsigned int to = graph[from].edges[j].edge;
					if(!mapping.count(to)){continue;}
					diag[i] += 1;
				}
			}
		}
		if(debug_level==9) cerr << getTime() << " [DEBUG:9]@float ini done" << endl;

		// Get max degree / sum of weights of nodes
		unsigned int max_d = max_of_diag((*nodes),diag,use_all_cores);	

		// Init randomized variables. 
		vector<float> x = generate_random_vector<float>(n);		

		// Orthogonalize + normalize vector + get initial lenght
		float current_length = 0;
		float last_length;

		(*x_hat) = makeOrthogonal<float>(x,use_all_cores);
		normalize(x_hat, &last_length, use_all_cores);

		// Repeat until difference < param_epsilon
		unsigned int iter = 0;	// catch huge clustering issues by keeping track here

		while(++iter < param_max_iter) { 

			if(debug_level==9) cerr << getTime() << " [DEBUG:9]@float iter:" << iter << endl;

			last_length = current_length;

			// Get a new x
			x = get_new_x<float>(x_hat, (*nodes), mapping, useWeights, use_all_cores);

			// Get y
			vector<float> y = getY<float>(max_d,*x_hat,&x,(*nodes),diag,use_all_cores);

			// Orthogonalize
			(*x_hat) = makeOrthogonal<float>(y,use_all_cores);

			// Get length (lambda) & normalize vector
			normalize(x_hat, &current_length, use_all_cores);

			if ( abs(current_length-last_length) < param_epsilon && iter >= min_iter ) break;	// prevent convergence by chance, converge to param_epsilon
		}
		if(debug_level==9) cerr << getTime() << " [DEBUG:9]@float post while" << endl;

		if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   " + to_string( iter ) + " / " + to_string( param_max_iter ) + " iterations required (error is " + to_string( abs(current_length-last_length) ) + ") @" +id_nodes );

		#ifdef DEBUG
			total_number_of_iterations_convergence+=iter;
		#endif

		if(useWeights){
			connectivity.value = abs( (-current_length+(double)2*max_d)/(double)(maxWeight*n) );
		}else{
			connectivity.value = abs( (-current_length+(double)2*max_d)/((double)n) );
		}
		//normalize(x_hat, &current_length, use_all_cores);
		if(debug_level>0) q->dispatch_log(getTime() +" [DEBUG]   Connectivity score @" +id_nodes+":" + to_string( connectivity.value )+ " = (-" + to_string(current_length)+"+2*"+to_string(max_d)+")/("+to_string(maxWeight)+"*"+to_string(n)+")" );

		// 5.17 catch hardly-converging groups
		if (iter >= param_max_iter) {
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Connectivity score of connected component with " + to_string( n ) + " elements did not converge perfectly in time." );
		}
		if(debug_level>0) q->dispatch_log( getTime() + " [DEBUG]   length:" + to_string( current_length ));
	}

	if (debug_level > 1){
		string log= getTime() +" [DEBUG]   Connectivity score @" +id_nodes+":" + to_string( connectivity.value );
		if ( (debug_level > 1 && (unsigned int)n<100 ) || debug_level > 2 ){
			log+= " eigenvector: (";
			for(unsigned int i = 0 ; i < (unsigned int)n ; i++)
				log+= to_string((*x_hat)[i]) + ","; 
			log+= ")";
		}
		log+= "\n";
		q->dispatch_log(log);
	}

	// Split groups if connectivity is too low, remove tree like structures that might have arosen
	if (param_con_threshold > 0 && abs(connectivity.value) < param_con_threshold) {
		
		// 5.17 new threshold option overwrites connectivity
		if (param_min_species >=1 ) {
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG:getConnectivity_float]  Start the calculation of the average gene/species score @" +id_nodes );
			float avg = calc_group(nodes);
			if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Found " + to_string( avg ) + " genes/species on average. User asked for at least " + to_string( param_min_species ) +" @" +id_nodes );
			if (avg <= param_min_species) {
				if (debug_level > 0) q->dispatch_log( getTime() + " [DEBUG]   Group is going to be accepted despite connectivity " + to_string( connectivity.value ) +" @" +id_nodes );
				// just to be safe from infinit loops due to rounding
				if (connectivity.value == 0) connectivity.value = 0.001;
				// no splitting despite bad connectivity
				connectivity.param_min_species_is_true=1;
				return connectivity;			
			}
		}
	}
	
	return connectivity;
}

bool comparator_pairfloatUInt ( const pair<float,unsigned int>& l, const pair<float,unsigned int>& r )
	 { return l.first < r.first; }

map<size_t,std::chrono::time_point<std::chrono::steady_clock> > t_tmp;

string getTime(void) {
	// returns the difference to the last call of this function (within the current thread)
	// used for debug
	size_t tid=std::hash<std::thread::id>{}(std::this_thread::get_id());
	if(!t_tmp.count(tid)){ t_tmp[tid]=std::chrono::steady_clock::now(); }
	string res = to_string(std::chrono::duration_cast<std::chrono::nanoseconds>( std::chrono::steady_clock::now( ) - t_tmp[tid] ).count()/1e+9);
	t_tmp[tid] = std::chrono::steady_clock::now( );
	return res;
}

#ifdef DEBUG
	////////////////////// Debug
	void debug__print_edgelist (protein& node, const unsigned int index, const int node_id) {
		cerr << node_id << ": ";	
		for (unsigned int j = 0; j < node.edges.size(); j++) {
			if (j == index) cerr << "*";
			cerr << node.edges[j].edge << " ";
		}
		cerr << "\n";
	}

	void debug__conn_integrity(vector<unsigned int>& nodes, float conn) {
		if (nodes.size() > 5) return;

		unsigned int sum = 0;
		for (unsigned int a = 0; a < nodes.size(); a++) {
			unsigned int from = nodes[a];
			sum += graph[from].edges.size();
		}

		sum /= 2;

		if (nodes.size() == 3) {
			if (sum == 2 && conn > 0.4) {
				cerr << "gs 3 with 2 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 3a").c_str() << "\n";throw;
			}
			if (sum == 3 && conn < 1) {
				cerr << "gs 3 with 3 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 3b").c_str() << "\n";throw;
			}
		}
		if (nodes.size() == 4) {
			if (sum == 3 && conn > 0.4) {
				cerr << "gs 4 with 3 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 4a").c_str() << "\n";throw;
			}
			if (sum < 6 && conn == 1) {
				cerr << "gs 4 with <6 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 4b").c_str() << "\n";throw;
			}
			if (sum == 6 && conn < 1) {
				cerr << "gs 4 with 6 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 4c").c_str() << "\n";throw;
			}
		}
		if (nodes.size() == 5) {
			if (sum == 4 && conn > 0.4) {
				cerr << "gs 5 with 4 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 5a").c_str() << "\n";throw;
			}
			if (sum < 10 && conn == 1) {
				cerr << "gs 5 with <10 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 5b").c_str() << "\n";throw;
			}
			if (sum == 10 && conn < 1) {
				cerr << "gs 5 with 10 edges had conn " << conn << "\n";
				cerr << string("integrity issue: connectivity of group size 5c").c_str() << "\n";throw;
			}
		}
	}

	void debug__graph_integrity(vector<unsigned int>& nodes) {
		// For each node
		for (unsigned int a = 0; a < nodes.size(); a++) {
			unsigned int from = nodes[a];

			// For each edge in + direction
			for (unsigned int i = 0; i < graph[from].edges.size(); i++) {
				unsigned int to = graph[from].edges[i].edge;

				if (to == from) {cerr << "ERROR: Edge from " << from << " to " << to << " is selfevident" << "\n"; cerr << string("integrity issue self hit").c_str() << "\n";throw;}

				// Check reverse direction
				// Foreach edge in - direction
				bool found = false;
				for (unsigned int j = 0; j < graph[to].edges.size(); j++) {
					unsigned int back = graph[to].edges[j].edge;
					if (back == from) {found = true; continue;}
				}
				if (!found) {
					cerr << "ERROR: Edge from " << from << " to " << to << " is unidirectional" << "\n"; cerr << string("integrity issue direction").c_str() << "\n";throw;
				}
			}
		}
	}

	/* Auxiliary routine: printing a matrix */
	void debug__print_matrix( int m, int n, float* a, int lda ) {
		int i, j;
		for( i = 0; i < m; i++ ) {
			for( j = 0; j < n; j++ ) printf( " %6.2f", a[i+j*lda] );
			printf( "\n" );
		}
	}
#endif

bool test__max_of_diag() {
	
	for(unsigned int j = 1 ; j < 100; j++){

		vector<unsigned int> nodes(j);
		for(unsigned int i = 0 ; i < nodes.size() ; i++) nodes[i] = i;
		
		vector<unsigned int> diag(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) diag[i] = i;

		if(max_of_diag(nodes,diag,false)!=j-1) return false;

	}

	{
		vector<unsigned int> nodes(10);
		for(unsigned int i = 0 ; i < nodes.size() ; i++) nodes[i] = i;
		
		vector<unsigned int> diag(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) diag[i] = 1;
		diag[0] = 2;

		if(max_of_diag(nodes,diag,false)!=2) return false;	
	}

	return true;
}

bool test__generate_random_vector() {
	
	for(unsigned int j = 1 ; j < 100; j++){
		std::vector<float> v = generate_random_vector<float>(j);
		for(unsigned int i = 0 ; i < v.size()-1 ; i++) if(v[i] == v[i+1]) return false;
	}
	return true;
}

bool test__get_new_x() {
	
	for(unsigned int j = 2 ; j < 100; j++){

		graph.clear();
		vector<protein>().swap(graph);
	
		vector<unsigned int> nodes(j);
		map<unsigned int,unsigned int> mapping; 
		for(unsigned int i = 0 ; i < nodes.size() ; i++){ nodes[i] = i; mapping[i]=i; }

		graph.resize(nodes.size());
		// create complete graph K_nodes.size() (all nodes are from nodes.size() different species) with edgeweight 10 
		for(unsigned int i = 0 ; i < nodes.size() ; i++){
			for(unsigned int k = 0 ; k < nodes.size() ; k++){
				wedge w;
				w.edge = i;
				w.weight = 10;
				graph[nodes[k]].edges.push_back(w); 
				graph[nodes[k]].species_id = k; 
				graph[nodes[k]].full_name = "X";
			}
		}

		vector<float> x(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) x[i] = i; 

		//test weighted case
		vector<float> x_new = get_new_x<float>(&x,nodes,mapping,1,false);
	
		if(x_new.size() != x.size())return false;
		for(unsigned int i = 0 ; i < nodes.size() ; i++) if(x_new[i]!=10*(nodes.size()-1)*(nodes.size())/2) return false;

		//test unweighted case
		x_new = get_new_x<float>(&x,nodes,mapping,0, false);
	
		if(x_new.size() != x.size())return false;
		for(unsigned int i = 0 ; i < nodes.size() ; i++) if(x_new[i]!=(nodes.size()-1)*(nodes.size())/2) return false;
	}

	{
		graph.clear();
		vector<protein>().swap(graph);
		vector<unsigned int> nodes(5);
		graph.resize(nodes.size());
		map<unsigned int,unsigned int> mapping; 
		for(unsigned int i = 0 ; i < nodes.size() ; i++){ 
			nodes[i] = i; mapping[i]=i; 
			graph[nodes[0]].species_id = 0; 
			graph[nodes[0]].full_name = "X";}

		/* 0--1--2
		**    |\
		**    3-4
		** edge weight is between (x,y) is the number xy (if x<y) else yx (not multiply, just concatenate)
		**/

		wedge w;
		
			w.edge = 1;
			w.weight = 10;
			graph[nodes[0]].edges.push_back(w); 
			w.edge = 0;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 13;
			graph[nodes[3]].edges.push_back(w); 
			w.edge = 3;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 14;
			graph[nodes[4]].edges.push_back(w); 
			w.edge = 4;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 3;
			w.weight = 34;
			graph[nodes[4]].edges.push_back(w); 
			w.edge = 4;
			graph[nodes[3]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 12;
			graph[nodes[2]].edges.push_back(w); 
			w.edge = 2;
			graph[nodes[1]].edges.push_back(w); 

		vector<float> x(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) x[i] = i; 

		//test weighted case
		vector<float> x_new = get_new_x<float>(&x,nodes,mapping,1,false);

		if(x_new.size() != x.size())return false;
		if(x_new[0]!=1*10) return false;
		if(x_new[1]!=0*10+2*12+3*13+4*14) return false;
		if(x_new[2]!=1*12) return false;
		if(x_new[3]!=1*13+4*34) return false;
		if(x_new[4]!=1*14+3*34) return false;

		//test unweighted case
		x_new = get_new_x<float>(&x,nodes,mapping,0,false);
	
		if(x_new.size() != x.size())return false;
		if(x_new[0]!=1) return false;
		if(x_new[1]!=0+2+3+4) return false;
		if(x_new[2]!=1) return false;
		if(x_new[3]!=1+4) return false;
		if(x_new[4]!=1+3) return false;
	}

	return true;
}

bool test__makeOrthogonal(){
	for(unsigned int j = 1 ; j < 100; j++){
		std::vector<float> x = generate_random_vector<float>(j);
		x=makeOrthogonal<float>(x,false);
		float sum=0;
		for (unsigned int i = 0; i < x.size(); i++) {sum += x[i];}
		//cerr << sum << "\n";
		if(sum > 1e-3 || sum < -1e-3)return false;
	}
	return true;
}

bool test__normalize(){
	for(unsigned int j = 1 ; j < 100; j++){
		std::vector<float> x = generate_random_vector<float>(j);
		float len=0;
		normalize(&x,&len,false);
		normalize(&x,&len,false); // second normalize should return a length of 1 
		if(len > 1+1e-3 || len < 1-1e-3)return false;
	}
	return true;
}

bool test__getY(){

	for(unsigned int j = 1 ; j < 100; j++){
		
		vector<unsigned int> nodes(j);
		for(unsigned int i = 0 ; i < nodes.size() ; i++) nodes[i]=i;
		
		vector<unsigned int> diag(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) diag[i]=i;

		float max_degree = max_of_diag(nodes,diag,false);

		vector<float> x_hat(nodes.size()),x_new(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++){ x_hat[i]=i; x_new[i] = -(float)i*(2*max_degree - diag[i]); }

		vector<float> y = getY<float>(max_degree,x_hat,&x_new,nodes,diag,false);
 
		if(y.size() != x_hat.size())return false;
		for(unsigned int i = 0 ; i < nodes.size() ; i++) if(y[i] > 1e-3 || y[i] < -1e-3)return false;
	}

	{
		vector<unsigned int> nodes(5);
		for(unsigned int i = 0 ; i < nodes.size() ; i++) nodes[i]=i;
		
		vector<unsigned int> diag(nodes.size());
		for(unsigned int i = 0 ; i < nodes.size() ; i++) diag[i]= (float)3;
		diag[3]=5;

		float max_degree = max_of_diag(nodes,diag,false); 

		vector<float> x_hat(nodes.size()),x_new(nodes.size(),false);
		x_hat[0]= (float)2.1; 
		x_hat[1]= (float)-3; 
		x_hat[2]= (float)10; 
		x_hat[3]= (float)9.5; 
		x_hat[4]= (float)0; 

		x_new[0] = (float)-14.7; //calculated by hand
		x_new[1] = (float)21;
		x_new[2] = (float)-70;
		x_new[3] = (float)-47.5;
		x_new[4] = (float)0;

		vector<float> y = getY<float>(max_degree,x_hat,&x_new,nodes,diag,false);

		for(unsigned int i = 0 ; i < nodes.size() ; i++) if(y[i] > 1e-3 || y[i] < -1e-3)return false;
	}

	return true;

}

bool test__lapack_power(bool do_lapack = true){
	{
		param_useWeights=1;
		param_double=1;

		graph.clear();
		vector<protein>().swap(graph);
		vector<unsigned int> nodes(5);
		graph.resize(nodes.size());
		map<unsigned int,unsigned int> mapping; 
		for(unsigned int i = 0 ; i < nodes.size() ; i++){ 
			nodes[i] = i; mapping[i]=i; 
			graph[nodes[0]].species_id = 0; 
			graph[nodes[0]].full_name = "X";}

		/* 0--1--2
		**    |\
		**    3-4
		** edge weight is between (x,y) is the number xy (if x<y) else yx (not multiply, just concatenate)
		**/

		wedge w;
		
			w.edge = 1;
			w.weight = 10;
			graph[nodes[0]].edges.push_back(w); 
			w.edge = 0;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 13;
			graph[nodes[3]].edges.push_back(w); 
			w.edge = 3;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 14;
			graph[nodes[4]].edges.push_back(w); 
			w.edge = 4;
			graph[nodes[1]].edges.push_back(w); 
			w.edge = 3;
			w.weight = 34;
			graph[nodes[4]].edges.push_back(w); 
			w.edge = 4;
			graph[nodes[3]].edges.push_back(w); 
			w.edge = 1;
			w.weight = 12;
			graph[nodes[2]].edges.push_back(w); 
			w.edge = 2;
			graph[nodes[1]].edges.push_back(w); 

		vector<double> x_hat;
		debug_level=0; // disable for getConnectivity_double, as otherwise the dynamic_queue is needed to dispatch logs
		connectivity conn = getConnectivity_double(&nodes,do_lapack,&x_hat,false,NULL);

		if( x_hat.size() != nodes.size() ){ return false; }

		// connectivity should be ~ 0.062
		if( conn.value > 0.08 || conn.value < 0.04 ){ return false; }

		// there should be a clear split of node 0 and node 3, if not then throw an error
		if( (x_hat[0] > 0 && x_hat[3] > 0) ||
			(x_hat[0] < 0 && x_hat[3] < 0) ) { return false; }
	}
	return true;
}

bool test__smirnovgrubb(){		
	// generate a 1000 sized vector with every value being 100 
	vector<double> test(1000,100);
	// add two outlier 1 & 2
	test.push_back(1);
	test.push_back(2);

	// the cutoff should include both outlier values => 2 

	return lowerSmirnovGrubbsTest_binary_search_cutoff(test)==2;
}
#endif /* _PROTEINORTHOCLUSTERING */