/*
   Copyright 2013-2015 Skytechnology sp. z o.o.
   Copyright 2023      Leil Storage OÜ

   This file is part of SaunaFS.

   SaunaFS is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, version 3.

   SaunaFS is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with SaunaFS. If not, see <http://www.gnu.org/licenses/>.
 */

#include "common/platform.h"

#include <fcntl.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <syslog.h>
#include <unistd.h>
#include <cerrno>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <thread>

#include "chunkserver/bgjobs.h"
#include "chunkserver/chunk_replicator.h"
#include "chunkserver/g_limiters.h"
#include "chunkserver/hdd_readahead.h"
#include "chunkserver/masterconn.h"
#include "chunkserver/network_main_thread.h"
#include "chunkserver/network_worker_thread.h"
#include "common/cwrap.h"
#include "common/event_loop.h"
#include "common/exceptions.h"
#include "common/massert.h"
#include "common/sockets.h"
#include "config/cfg.h"
#include "devtools/TracePrinter.h"
#include "protocol/SFSCommunication.h"
#include "slogger/slogger.h"

static int lsock;
static int32_t lsockpdescpos;

std::list<std::thread> networkThreads;
std::list<NetworkWorkerThread> networkThreadObjects;
std::list<NetworkWorkerThread>::iterator nextNetworkThread;

static uint32_t mylistenip;
static uint16_t mylistenport;

// from config
static char *ListenHost;
static char *ListenPort;
static uint32_t gNrOfNetworkWorkers;
static uint32_t gNrOfHddWorkersPerNetworkWorker;
static uint32_t gBgjobsCountPerNetworkWorker;

static std::atomic<bool> gDoTerminate = false;

bool doTerminate() {
	return gDoTerminate.load();
}

void chunkReplicatorReload() {
	unsigned rep_total = cfg_get_minmaxvalue<unsigned>("REPLICATION_TOTAL_TIMEOUT_MS",
	                                                   ChunkReplicator::kDefaultTotalTimeout_ms,
	                                                   1000, 60 * 60 * 1000);
	unsigned rep_wave = cfg_get_minmaxvalue<unsigned>("REPLICATION_WAVE_TIMEOUT_MS",
	                                                  ChunkReplicator::kDefaultWaveTimeout_ms,
	                                                  50, 30 * 1000);
	unsigned rep_connection = cfg_get_minmaxvalue<unsigned>("REPLICATION_CONNECTION_TIMEOUT_MS",
	                                                        ChunkReplicator::kDefaultConnectionTimeout_ms,
	                                                        200, 30 * 1000);

	gReplicator.setTotalTimeout(rep_total);
	gReplicator.setWaveTimeout(rep_wave);
	gReplicator.setConnectionTimeout(rep_connection);
}

void replicationBandwidthLimitReload() {
	if (cfg_isdefined("REPLICATION_BANDWIDTH_LIMIT_KBPS")) {
		replicationBandwidthLimiter().setLimit(cfg_getuint32("REPLICATION_BANDWIDTH_LIMIT_KBPS", 0));
	} else {
		replicationBandwidthLimiter().unsetLimit();
	}
}

void mainNetworkThreadReload(void) {
	TRACETHIS();

	cfg_warning_on_value_change("NR_OF_NETWORK_WORKERS", gNrOfNetworkWorkers);
	cfg_warning_on_value_change("NR_OF_HDD_WORKERS_PER_NETWORK_WORKER",
	                            gNrOfHddWorkersPerNetworkWorker);
	cfg_warning_on_value_change("BGJOBSCNT_PER_NETWORK_WORKER",
	                            gBgjobsCountPerNetworkWorker);

	try {
		replicationBandwidthLimitReload();
	} catch (std::exception& ex) {
		safs_pretty_errlog(LOG_ERR,
				"main server module: can't reload REPLICATION_BANDWIDTH_LIMIT_KBPS: %s",
				ex.what());
	}
	chunkReplicatorReload();

	gMaxBlocksPerHddWriteJob = cfg_get_minmaxvalue<uint16_t>(
	    "MAX_BLOCKS_PER_HDD_WRITE_JOB", NetworkWorkerThread::kDefaultMaxBlocksPerHddWriteJob,
	    NetworkWorkerThread::kMinBlocksPerHddWriteJob,
	    NetworkWorkerThread::kMaxBlocksPerHddWriteJob);
	gMaxBlocksPerHddReadJob = cfg_get_minvalue<uint16_t>(
	    "MAX_BLOCKS_PER_HDD_READ_JOB", NetworkWorkerThread::kDefaultMaxBlocksPerHddReadJob, 1);
	gMaxParallelHddReadJobsPerCsEntry = cfg_get_minvalue<uint16_t>(
	    "MAX_PARALLEL_HDD_READ_JOBS_PER_CS_ENTRY",
	    NetworkWorkerThread::kDefaultMaxParallelHddReadJobsPerCsEntry, 1);

	size_t maxBuffersPoolSize_mb = cfg_get_minvalue<size_t>("MAX_BUFFERS_POOL_SIZE_MB", 512, 0);
	setNewMaxIoBuffersPoolSize(maxBuffersPoolSize_mb);

	gHDDReadAhead.setReadAhead_kB(
			cfg_get_maxvalue<uint32_t>("READ_AHEAD_KB", 0, SFSCHUNKSIZE / 1024));
	gHDDReadAhead.setMaxReadBehind_kB(
			cfg_get_maxvalue<uint32_t>("MAX_READ_BEHIND_KB", 0, SFSCHUNKSIZE / 1024));

	char *oldListenHost, *oldListenPort;
	int newlsock;
	oldListenHost = ListenHost;
	oldListenPort = ListenPort;
	ListenHost = cfg_getstr("CSSERV_LISTEN_HOST", "*");
	ListenPort = cfg_getstr("CSSERV_LISTEN_PORT", "9422");
	if (strcmp(oldListenHost, ListenHost) == 0 && strcmp(oldListenPort, ListenPort) == 0) {
		free(oldListenHost);
		free(oldListenPort);
		safs_pretty_syslog(LOG_NOTICE,
				"main server module: socket address hasn't changed (%s:%s)",
				ListenHost, ListenPort);
		return;
	}

	newlsock = tcpsocket();
	if (newlsock < 0) {
		safs_pretty_errlog(LOG_WARNING,
				"main server module: socket address has changed, but can't create new socket");
		free(ListenHost);
		free(ListenPort);
		ListenHost = oldListenHost;
		ListenPort = oldListenPort;
		return;
	}
	tcpnonblock(newlsock);
	tcpnodelay(newlsock);
	tcpreuseaddr(newlsock);
	if (tcpsetacceptfilter(newlsock) < 0 && errno != ENOTSUP) {
		safs_silent_errlog(LOG_NOTICE, "main server module: can't set accept filter");
	}
	if (tcpstrlisten(newlsock, ListenHost, ListenPort, 100) < 0) {
		safs_pretty_errlog(LOG_ERR,
				"main server module: socket address has changed, but can't listen on socket (%s:%s)",
				ListenHost, ListenPort);
		free(ListenHost);
		free(ListenPort);
		ListenHost = oldListenHost;
		ListenPort = oldListenPort;
		tcpclose(newlsock);
		return;
	}
	safs_pretty_syslog(LOG_NOTICE,
			"main server module: socket address has changed, now listen on %s:%s",
			ListenHost, ListenPort);
	free(oldListenHost);
	free(oldListenPort);
	tcpclose(lsock);
	lsock = newlsock;
}

void mainNetworkThreadDesc(std::vector<pollfd> &pdesc) {
	TRACETHIS();
	if (doTerminate()) {
		return;
	}

	pdesc.push_back({lsock, POLLIN, 0});
	lsockpdescpos = pdesc.size() - 1;
}

void mainNetworkThreadWantExit(void) {
	TRACETHIS();
	safs::log_info("closing {}:{}", ListenHost, ListenPort);
	// Closing the listening socket will cause the main thread to stop accepting new connections and
	// eventually exit after processing existing ones.
	tcpclose(lsock);

	free(ListenHost);
	free(ListenPort);

	// Ask worker threads to terminate and close their connections. They will be forcefully
	// terminated after a timeout if they don't exit on their own.
	for (auto& threadObject : networkThreadObjects) {
		threadObject.askForTermination();
	}

	gDoTerminate.store(true);
}

bool networkThreadsCanExit() {
	TRACETHIS();
	bool allTerminated = true;
	for (auto &threadObject : networkThreadObjects) {
		if (!threadObject.updateAndCheckTerminationStatus()) { allTerminated = false; }
	}
	return allTerminated;
}

int mainNetworkThreadCanExit() {
	// Preserve this order:
	// networkThreadsCanExit() must be checked before masterconn_canexit().
	// If masterconn_canexit() is checked first, a network worker may still be processing an
	// endChunkLock, which could add statuses to the masterconn job pool after masterconn_canexit()
	// returns true. This could lead to the chunkserver exiting prematurely while holding chunk
	// locks that have not been replied to the master.
	return networkThreadsCanExit() && masterconn_canexit();
}

void mainNetworkThreadTerm(void) {
	TRACETHIS();

	for (auto &thread : networkThreads) {
		if (thread.joinable()) { thread.join(); }
	}

	networkThreads.clear();
	networkThreadObjects.clear();
}

void mainNetworkThreadServe(const std::vector<pollfd> &pdesc) {
	TRACETHIS();
	if (doTerminate()) {
		return;
	}

	int newSocketFD;

	if (lsockpdescpos >= 0 && (pdesc[lsockpdescpos].revents & POLLIN)) {
		newSocketFD = tcpaccept(lsock);
		if (newSocketFD < 0) {
			safs_silent_errlog(LOG_NOTICE, "accept error");
		} else {
			if (nextNetworkThread == networkThreadObjects.end()) {
				nextNetworkThread = networkThreadObjects.begin();
			}
			if (nextNetworkThread->backgroundJobPool()->getJobCount()
					>= (gBgjobsCountPerNetworkWorker * 9) / 10) {
				safs_pretty_syslog(LOG_WARNING, "jobs queue is full !!!");
				tcpclose(newSocketFD);
			} else {
				nextNetworkThread->addConnection(newSocketFD);
			}
			++nextNetworkThread;
		}
	}
}

int mainNetworkThreadInit(void) {
	TRACETHIS();
	ListenHost = cfg_getstr("CSSERV_LISTEN_HOST", "*");
	ListenPort = cfg_getstr("CSSERV_LISTEN_PORT", "9422");

	gNrOfNetworkWorkers = cfg_get_minvalue<uint32_t>(
	    "NR_OF_NETWORK_WORKERS",
	    NetworkWorkerThread::kDefaultNumberOfNetworkWorkers, 1);
	gNrOfHddWorkersPerNetworkWorker = cfg_get_minvalue<uint32_t>(
	    "NR_OF_HDD_WORKERS_PER_NETWORK_WORKER",
	    NetworkWorkerThread::kDefaultNumberOfHddWorkersPerNetworkWorker, 1);
	gBgjobsCountPerNetworkWorker = cfg_get_minvalue<uint32_t>(
	    "BGJOBSCNT_PER_NETWORK_WORKER",
	    NetworkWorkerThread::kDefaultMaxBackgroundJobsPerNetworkWorker, 10);
	std::string ioPriorityModeStr = cfg_getstring("IO_PRIORITY_MODE", "FIFO");
	if (ioPriorityModeStr == "SWITCH") {
		// Must clearly say that the mode is Switch, otherwise it will be Fifo. This is because Fifo
		// is the default and more tested mode.
		gIOPriorityMode = IOPriorityMode::Switch;
	} else {
		gIOPriorityMode = IOPriorityMode::Fifo;
		if (ioPriorityModeStr != "FIFO") {
			safs::log_warn("Invalid IO_PRIORITY_MODE '{}', defaulting to FIFO", ioPriorityModeStr);
		}
	}

	gMaxBlocksPerHddWriteJob = cfg_get_minmaxvalue<uint16_t>(
	    "MAX_BLOCKS_PER_HDD_WRITE_JOB", NetworkWorkerThread::kDefaultMaxBlocksPerHddWriteJob,
	    NetworkWorkerThread::kMinBlocksPerHddWriteJob,
	    NetworkWorkerThread::kMaxBlocksPerHddWriteJob);
	gMaxBlocksPerHddReadJob = cfg_get_minvalue<uint16_t>(
	    "MAX_BLOCKS_PER_HDD_READ_JOB", NetworkWorkerThread::kDefaultMaxBlocksPerHddReadJob, 1);
	gMaxParallelHddReadJobsPerCsEntry = cfg_get_minvalue<uint16_t>(
	    "MAX_PARALLEL_HDD_READ_JOBS_PER_CS_ENTRY",
	    NetworkWorkerThread::kDefaultMaxParallelHddReadJobsPerCsEntry, 1);

	size_t maxBuffersPoolSize_mb = cfg_get_minvalue<size_t>("MAX_BUFFERS_POOL_SIZE_MB", 512, 0);
	setNewMaxIoBuffersPoolSize(maxBuffersPoolSize_mb);

	gHDDReadAhead.setReadAhead_kB(
			cfg_get_maxvalue<uint32_t>("READ_AHEAD_KB", 0, SFSCHUNKSIZE / 1024));
	gHDDReadAhead.setMaxReadBehind_kB(
			cfg_get_maxvalue<uint32_t>("MAX_READ_BEHIND_KB", 0, SFSCHUNKSIZE / 1024));

	lsock = tcpsocket();
	if (lsock < 0) {
		throw InitializeException("main server module: can't create socket :" +
				errorString(errno));
	}
	tcpnonblock(lsock);
	tcpnodelay(lsock);
	tcpreuseaddr(lsock);
	if (tcpsetacceptfilter(lsock) < 0 && errno != ENOTSUP) {
		safs_silent_errlog(LOG_NOTICE, "main server module: can't set accept filter");
	}
	tcpresolve(ListenHost, ListenPort, &mylistenip, &mylistenport, 1);
	if (tcpnumlisten(lsock, mylistenip, mylistenport, 100) < 0) {
		throw InitializeException("main server module: can't listen on socket" +
				errorString(errno));
	}
	safs_pretty_syslog(LOG_NOTICE, "main server module: listen on %s:%s", ListenHost, ListenPort);

	eventloop_reloadregister(mainNetworkThreadReload);
	eventloop_wantexitregister(mainNetworkThreadWantExit);
	eventloop_canexitregister(mainNetworkThreadCanExit);
	eventloop_destructregister(mainNetworkThreadTerm);
	eventloop_pollregister(mainNetworkThreadDesc, mainNetworkThreadServe);

	try {
		replicationBandwidthLimitReload();
	} catch (Exception& e) {
		throw InitializeException("can't initialize replication bandwidth limiter: " + e.message());
	}
	chunkReplicatorReload();

	return 0;
}

int mainNetworkThreadInitThreads(void) {
	for (uint32_t i = 0; i < gNrOfNetworkWorkers; ++i) {
		networkThreadObjects.emplace_back(i, gNrOfHddWorkersPerNetworkWorker,
				gBgjobsCountPerNetworkWorker);
	}
	for (auto obj = networkThreadObjects.begin(); obj != networkThreadObjects.end(); ++obj) {
		networkThreads.push_back(std::thread(std::ref(*obj)));
	}
	sassert(!networkThreads.empty());
	nextNetworkThread = networkThreadObjects.end();
	return 0;
}

uint32_t mainNetworkThreadGetListenIp() {
	TRACETHIS();
	return mylistenip;
}

uint16_t mainNetworkThreadGetListenPort() {
	TRACETHIS();
	return mylistenport;
}
