#!/usr/bin/env bds

#-------------------------------------------------------------------------------
#
# Create databases
#
#-------------------------------------------------------------------------------


# Command line options
help Build and test options
bundleLists			:= false		help Create bundle lists and config entries. OBSOLETE
createConfig		:= false		help Create config file
createDocs			:= false		help Create documentation web pages from markdown
db					:= false		help Build databases
string[] dbs						help Build databases
dbTest				:= false		help Build databases used in test cases
distro	 			:= false		help Create distribution files (zip)
distroBundles		:= false		help Create bundle files (zip)
distroCore 			:= false		help Create distribution files (zip only 'core')
make	 			:= false		help Make (build JAR files)
test	 			:= false		help Run SnpEff / SnpSift test cases

help Data download options
download 			:= false		help Download all genomes
downloadSet			:= ''			help Download one genomes set

help Upload options
uploadCore			:= false		help Upload 'core' package
uploadDbNsfp		:= false		help Upload dbNsfp to google drive
uploadDbs			:= false		help Upload all databases in 'zip' directory
uploadDev			:= false		help Upload 'development' version
string[] zipGenome	= []			help Zip a specific genome

help Azure options
azureKeysFile		:= "$HOME/snpEff/.azure"					help Azure credentials file
azureBlobEndpoint	:= 'https://snpeff.blob.core.windows.net'	help Azure Blob entry point
azureContainer		:= 'databases'								help Azure container for databases
azureQueryString	:= ''										help Azure 'query string' containing a security token.

# Process us one CPU
cpus = 1

# Versions
help Ensembl
downloadEnsembl			:= ''			help Download Ensemlb genome, e.g. 'GRCh38.103'. Note: You also must to specify 'downloadEnsemblSpecies'
downloadEnsemblSpecies	:= ''			help Ensembl species to download, e.g. 'Homo_sapiens', Note: It must be capitalized exactly as Ensembl defines it
ensemblBfmppRelease		:= 52			help Ensembl BFMPP release number
ensemblRelease			:= 105			help Vertebrate release number
#ensemblReleaseGrch37	:= 75			help Final release for Ensembl's GRCh37
ensemblReleases			:= [ensemblRelease]

help MANE 
downloadMane			:= false		help Download MANE
maneGenome				:= 'GRCh38'		help MANE genome
maneRelease				:= '0.95'		help MANE release version
maneTrIdType			:= 'ensembl'	help MANE release transcript ID type: {'ensembl', 'refseq'}

help UCSC
downloadUcsc		:= ''			help Download genome files from UCSC (genome name)

help NCBI
downloadNcbi		:= ''			help Download genome files from NCBI (both 'genome name' and 'genome ID' are required)
ncbiId				:= ''			help Download genome ID from NCBI (both 'genome name' and 'genome ID' are required)

help Flybase
flybaseRelease		:= "FB2021_06"	help Flybase release
#flybaseGenomes		:= [ 'dana_r1.06', 'dere_r1.05', 'dgri_r1.05', 'dmel_r6.43', 'dmoj_r1.04', 'dper_r1.3', 'dpse_r3.04', 'dsec_r1.3', 'dsim_r2.02', 'dvir_r1.07', 'dwil_r1.05', 'dyak_r1.05' ]
flybaseGenomes		:= [ 'dmel_r6.43']	# Note: It looks like FlyBase is not providing any GTF files for other than 'dmel', this is really weird, they used to have them in the past (maybe the project is dead/dying?)

# Human
GRCH				:= "GRCh38"
GRCHs				:= ["GRCh37", "GRCh38"]
GRCh2Hg				:= {"GRCh37" => "hg19", "GRCh38" => "hg38"}
humanChrs			:= [ '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '21', '22', 'X', 'Y', 'MT']
humanChrsDd			:= [ '01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19', '21', '22', 'X', 'Y', 'MT']
dbNsfpVersion		:= '4.2a'
# UCSC full list of genomes from "curl ftp://hgdownload.cse.ucsc.edu/goldenPath/"
ucscGenomes			:= ['ailMel1', 'allMis1', 'anoCar2', 'anoGam3', 'apiMel2', 'aplCal1', 'aptMan1', 'balAcu1', 'bisBis1', 'bosTau9', 'braFlo1', 'caeJap1', 'caePb2', 'caeRem3', 'calJac4', 'calMil1', 'canFam6', 'cavPor3', 'cb3', 'ce11', 'ce2', 'ce4', 'ce6', 'cerSim1', 'chlSab2', 'choHof1', 'chrPic1', 'ci3', 'criGri1', 'criGriChoV1', 'criGriChoV2', 'danRer11', 'dasNov3', 'dipOrd1', 'dm6', 'dp3', 'droAna2', 'droEre1', 'droGri1', 'droMoj2', 'droPer1', 'droSec1', 'droSim1', 'droVir2', 'droYak2', 'eboVir3', 'echTel2', 'enhLutNer1', 'equCab3', 'eriEur2', 'felCat9', 'fr3', 'gadMor1', 'galGal6', 'galVar1', 'gasAcu1', 'geoFor1', 'gorGor6', 'hetGla2', 'hg19', 'hg38', 'latCha1', 'loxAfr3', 'macEug2', 'macFas5', 'manPen1', 'melGal5', 'melUnd1', 'micMur2', 'mm9', 'mm10', 'mm39', 'mm9', 'monDom5', 'musFur1', 'nanPar1', 'nasLar1', 'neoSch1', 'nomLeu3', 'ochPri3', 'oreNil2', 'ornAna2', 'oryCun2', 'oryLat2', 'otoGar3', 'oviAri4', 'panPan3', 'panTro6', 'papAnu4', 'papHam1', 'petMar3', 'ponAbe3', 'proCap1', 'pteVam1', 'rheMac10', 'rhiRox1', 'rn7', 'sacCer3', 'saiBol1', 'sarHar1', 'sorAra2', 'speTri2', 'strPur2', 'susScr11', 'taeGut2', 'tarSyr2', 'tetNig2', 'thaSir1', 'triMan1', 'tupBel1', 'turTru2', 'vicPac2', 'wuhCor1', 'xenLae2', 'xenTro10' ]
# Removed:
#       aquChr2: Missing CDS fasta
ucscGenomesRefGene	:= [ 'hg19', 'hg38', 'mm9', 'mm10', 'mm39' ]
ucscGenomesKg		:= [ 'hg19', 'hg38', 'mm9', 'mm10', 'mm39' ]

help Compatible databases
dbCompatibleVersions := ['5.0'] help List of database compatible versions

# Mouse
GRCM				:= "GRCm38"

# Configuration parameters
bundles				:= ["ENSEMBL_BFMPP_$ensemblBfmppRelease"]

# Commands
snpeff				:= "java -Xmx10G -jar snpEff.jar "
snpeffBuild			:= "$snpeff build -v"
snpsift				:= "java -jar SnpSift.jar "
wgetCmd				:= "wget -N --retr-symlinks"
cp					:= "cp -v -u"

# Directories
dirSnpEff			:= "$HOME/snpEff"
dirBuild			:= "$dirSnpEff/build"
dirData				:= "$dirSnpEff/data"
dirDb				:= "$dirSnpEff/db"
dirDarned			:= "$dirDb/darned"
dirDownload			:= "$dirSnpEff/download"
dirNextProt         := "$dirDb/nextProt"
dirPdb              := "$dirDb/pdb"
dirScripts			:= "$dirSnpEff/scripts"
dirScriptsBuild		:= "$dirSnpEff/scripts_build"
dirZip				:= "$dirSnpEff/zip"
pwmsBin				:= "$dirDb/jaspar/pwms.bin"
ensembl2refSeqTrIds := "$dirDb/ensembl2refSeq/ensembl2refSeq.GRCh.txt"

# URLs
snpeffDownload		:= "$azureBlobEndpoint/versions/snpEff_latest_core.zip"

# Output files
buildSummary		:= "build.out"

# Include (depends on some of the variables defined above)
include 'dbNsfp'
include 'download/download.bds'
include 'util'

#-------------------------------------------------------------------------------
# Variables
#-------------------------------------------------------------------------------

string{} built

# SnpEff version
snpEffVersion := ""

#-------------------------------------------------------------------------------
# Add a task to build a database for 'genome'
#-------------------------------------------------------------------------------
bool addBuildDb(string genome) {
	dir := "$dirData/$genome"
	db := "$dir/snpEffectPredictor.bin"
	geneFiles := dir.dirPath("**/genes.*") 

	# No 'genes' file? Error
	if( geneFiles.isEmpty() ) {
		warning("No genes file for $genome\n")
		return( false );
	}

	geneFile := geneFiles.head()
	if( built.hasKey(genome) ) {
		log("Genome $genome already built\n")
	} else if( db <- geneFile ) {
		print("BUILD:\t$genome\tgeneFile: $geneFile\tdb: $db\n")

		# Use 'storeSeqs' for human and mouse genome
		opt := ( genome.startsWith('GRC') || genome.startsWith('hg') ? "-storeSeqs" : "" )

		task $snpeffBuild $opt $genome 2>&1 | tee $dirBuild/build.$genome\.out
	} else {
		print("OK  :\t$genome\tgeneFile: $geneFile\tdb: $db\n")
	}

	# Mark as built
	built{genome} = genome
	return( true )
}

#-------------------------------------------------------------------------------
# Build JAR files (compile)
#-------------------------------------------------------------------------------
void build() {
	buildConfig()

	println("Building SnpEff & SnpSift")
	sys ./scripts_build/make.sh
}

#-------------------------------------------------------------------------------
# Build bundle lists
#-------------------------------------------------------------------------------
void buildBundleLists() {
	println "Make bundle lists:"

	for( string bundle : bundles ) {
		conf := "config/snpEff.bundles.$bundle\.config"

		if( conf.exists() ) {
			print "Config file '$conf' exists, nothing done\n"
		} else {
			println "Creating bundles '$bundle' -> '$conf'"
			sys cat config/snpEff.$bundle\.config | cut -f 1 -d : | grep ".genome" | sed "s/.genome\s*//" | tr -d " " > snpEff.bundles.$bundle\.genomes.txt
			sys ./scripts_build/createBundles.pl snpEff.bundles.$bundle\.genomes.txt $bundle > $conf
		}
	}

	# We must rebuild config file
	buildConfig()
}

#-------------------------------------------------------------------------------
# Build configuration file (snpEff.config)
#-------------------------------------------------------------------------------
void buildConfig() {
	confDir := "$dirSnpEff/config"
	conf := "$dirSnpEff/snpEff.config"

	println "Building config file '$conf'"

	conf.rm()
	for(string c: ["snpEff.core.config", "snpEff.dbs.config", "snpEff.test.config"] ) {
		println("Adding config '$c'")
		sys cat '$confDir/$c' >> '$conf'
	}

	for(int release : ensemblReleases ) {
		sys echo >> '$conf'
		sys echo "#---" >> '$conf'
		sys echo "# ENSEMBL release $release" >> '$conf'
		sys echo "#---" >> '$conf'
		c := "$confDir/snpEff.ENSEMBL_$release\.config"
		println("Adding config '$c'")
		sys cat '$c' >> '$conf'
	}

	sys echo >> '$conf'
	sys echo "#---" >> '$conf'
	sys echo "# NCBI genomes" >> '$conf'
	sys echo "#---" >> '$conf'
	println("Adding configs '$confDir/snpEff.NCBI.*.config'")
	sys cat	$confDir/snpEff.NCBI.*.config >> '$conf'

	sys echo >> '$conf'
	sys echo "#---" >> '$conf'
	sys echo "# ENSEMBL BFMPP release $ensemblBfmppRelease" >> '$conf'
	sys echo "#---" >> '$conf'
	c := "$confDir/snpEff.ENSEMBL_BFMPP_$ensemblBfmppRelease\.config"
	println("Adding config '$c'")
	sys cat "$c" >> '$conf'

	sys echo >> '$conf'
	sys echo "#---" >> '$conf'
	sys echo "# FLYBASE release $flybaseRelease" >> '$conf'
	sys echo "#---" >> '$conf'
	c = "$confDir/snpEff.FLYBASE_$flybaseRelease\.config"
	println("Adding config '$c'")
	sys cat	'$c' >> '$conf'

	for( string b : bundles) {
		sys echo >> '$conf'
		sys echo "#---" >> '$conf'
		sys echo "# ENSEMBL Bundles $b" >> '$conf'
		sys echo "#---" >> '$conf'
		c = "$confDir/snpEff.bundles.$b\.config"
		println("Adding config '$c'")
		sys cat '$c' >> '$conf'
	}

	sys echo >> '$conf'
	sys echo "#---" >> '$conf'
	sys echo "# Custom requests " >> '$conf'
	sys echo "#---" >> '$conf'
	c = "$confDir/snpEff.custom_requests.config"
	println("Adding config '$c'")
	sys cat	'$c' >> '$conf'

	sys echo >> '$conf'
	sys echo "#---" >> '$conf'
	sys echo "# COVID19 " >> '$conf'
	sys echo "#---" >> '$conf'
	c = "$confDir/snpEff.covid19.config"
	println("Adding config '$c'")
	sys cat	'$c' >> '$conf'

	for(string dbv: dbCompatibleVersions) {
        sys echo >> '$conf'
        sys echo "#---" >> '$conf'
        sys echo "# Compatible versions: $dbv " >> '$conf'
        sys echo "#---" >> '$conf'
        c = "$confDir/snpEff.version.$dbv\.config"
        println("Adding config '$c'")
        sys cat	'$c' >> '$conf'
	}
}

#-------------------------------------------------------------------------------
# Build all databases
#-------------------------------------------------------------------------------
void buildDb() {
	# We must do this before building
	copyPwm()
	copyCytoBands()

	if( ! dbs.isEmpty() )	dbs.buildDbs()			# Build some genomes
	else buildDbAll()	# Build ALL genomes

	buildDbSpecial()
	buildDbPdb()
}

#-------------------------------------------------------------------------------
# Build all databases
#-------------------------------------------------------------------------------
bool buildDbAll() {
	print("Build all dbs: Start\n")
	dirBuild.mkdir()	# Make sure build dir exists
	
	# Look into all directories
	print("Available databases:\n")
	dbids := sys $snpeff databases | cut -f 1 | tail -n +3 

	print("Building:\n")
	for(string genome : dbids.lines()  ) {
		genome = genome.trim().baseName()
		addBuildDb(genome)
	}
	wait

	# Create build summary
	print("Build: Checking build logs!\n")
	sys cat $dirBuild/build.*.out | ./scripts_build/buildCheck.pl | tee $buildSummary

	print("Build: Done!\n")
	return( true )
}

#-------------------------------------------------------------------------------
# Build some databases
#-------------------------------------------------------------------------------
bool buildDbs(string[] dbids) {
	print("Build dbs: Start\n")
	dirBuild.mkdir()	# Make sure build dir exists
	
	print("Building:\n")
	for(string genome : dbids  ) {
		println "\t$genome"
		addBuildDb(genome)
	}
	wait

	print("Build: Done!\n")
	return( true )
}

#-------------------------------------------------------------------------------
# Build 'interaction' databases
#-------------------------------------------------------------------------------
void buildDbPdb() {
	print("Build PDB.\n")

	dataDirsHg := dirsHg()

	# Check each genome version
	for( string hg : dataDirsHg ) {
		hg = hg.baseName()
		println "\t$hg"
		pdbfile := "$dirData/$hg/interactions.bin"

		if( !pdbfile.exists() ) {
			print("PDB ($hg): Build\t\tPDB db file: '$pdbfile'\n")
			task $snpeff pdb -v -pdbDir $dirPdb -idMap $dirPdb/idMap_pdbId_ensemblId_refseqId.txt.gz $hg 
		} else {
			print("PDB ($hg): OK\t\tPDB db file: '$pdbfile'\n")
		}
	}
}

#-------------------------------------------------------------------------------
# Build special databases
#-------------------------------------------------------------------------------
void buildDbSpecial() {
	buildDbNextProt()
}

#-------------------------------------------------------------------------------
# Build databases used in test cases
#-------------------------------------------------------------------------------
void buildDbTest() {
	dirs := "$dirSnpEff/data/".dir("test*")
	dirs.remove('testNM_015296')	# This database is supposed to build with an error in one of the test cases

	# This one needs PWMs
	pwmsDst := "data/testHg3770Chr22/" + pwmsBin.baseName()
	if( pwmsDst <- pwmsBin ) sys cp $pwmsBin $pwmsDst

	# Build all test genomes
	for( string gen : dirs ) {
		db := "$dirSnpEff/data/$gen/snpEffectPredictor.bin"
		if( ! db.exists() )	{
			println "Genome: $gen\tdb: '$db'"
			cdsFasta := "$dirSnpEff/data/$gen/cds.fa.gz"
			proteinFasta := "$dirSnpEff/data/$gen/protein.fa.gz"
			checkCdsFlag := (cdsFasta.exists() ? '' : '-noCheckCds')
			checkProteinFlag := (proteinFasta.exists() ? '' : '-noCheckProtein')
			task( cpus := 2 ) {
				sys $snpeff build -v -storeSeqs $checkCdsFlag $checkProteinFlag $gen 2>&1 | tee $gen\.build
			}
		}
	}

	# All databases must be built before next sections
	wait

	# Build PDB database for testHg19Chr1
	# We need to make sure that PDB entries for test cases are in these dirs
	pdbTestCases := ['testHg19Chr1', 'testHg19Chr22']
	for( string genome : pdbTestCases ) {
		interactions := "$dirSnpEff/data/$genome/interactions.bin"
		if( !interactions.exists() ) {
			task $snpeff pdb -v \
					-pdbDir $dirSnpEff/data/$genome \
					-idMap $dirPdb/idMap_pdbId_ensemblId_refseqId.txt.gz \
					$genome
		}
	}

	# Build nextprot databases
	nextProtTestCases := ['testHg3770Chr22']
	for( string genome : nextProtTestCases ) {
		nextProtDir := "$dirSnpEff/tests/integration/nextProt/nextProt"
		nextProtDb := "$dirSnpEff/data/$genome/nextProt.bin"
		task(!nextProtDb.exists()) sys $snpeff buildNextProt -v $genome '$nextProtDir'
	}
}

#-------------------------------------------------------------------------------
# Build special databases
#-------------------------------------------------------------------------------
void buildDbNextProt() {
	# Note: This takes a huge amount of memory, don't run multiple of these process because the server will crash
	print("Build NextProt\n")

	# XML files used to build NextProt
	dirNextProt := "$dirSnpEff/db/nextProt/"
	xmlFiles := dirNextProt.dirPath("**/*.xml")

	# Check each genome version
	for( string hg : dirsHg() ) {
		hg = hg.baseName()
		npfile := "$dirData/$hg/nextProt.bin"

		if( !npfile.exists() ) {
			print("NextProt ($hg): Build\t\tNextProt db file: '$npfile'\n")
			sys $snpeff buildNextProt -v -trIds $ensembl2refSeqTrIds $hg db/nextProt/
		} else {
			print("NextProt ($hg): OK\t\tNextProt db file: '$npfile'\n")
		}
	}
}

#-------------------------------------------------------------------------------
# Create documenation web pages from markdown
#-------------------------------------------------------------------------------
void buildDocs() {
	println "Building documentation"

	# Cleanup old 'site' directory 
	sys rm -rvf '$dirSnpEff/site'
	
	# Build with mkdocs
	sys cd '$dirSnpEff' ; source ./bin/activate ; ./bin/mkdocs build

	# Copy 'index.html' 
	sys cp -vf '$dirSnpEff/src/docs/index.html' '$dirSnpEff/site/'

	# Cleanup old docs
	sys rm -vf '$dirSnpEff/docs/*/index.html'

	# Copy new pages
	sys cp -rvf $dirSnpEff/site/* $dirSnpEff/docs/
}

#-------------------------------------------------------------------------------
# Copy CytoBands to dirs
#-------------------------------------------------------------------------------
void copyCytoBands() {
	humanDirs := dirsHg()
	print("Copying cytoBands file to human genomes\n")
	for( string hg : humanDirs ) {
		dir := "$dirData/$hg/"
		gr := grch(hg)
		cyto := "$dirDb/$gr/cytoBand/cytoBand.txt.gz"
		print("\t$cyto\t=>\t$dir\n")

		sys cp $cyto $dir
	}
}

#-------------------------------------------------------------------------------
# Copy pwms.bin to every human dir
#-------------------------------------------------------------------------------
void copyPwm() {
	humanDirs := dirsHg()
	print("Copying $pwmsBin to human genomes\n")
	for( string hg : humanDirs ) {
		dir := "$dirData/$hg/"
		print("\t$hg: $dir\n")
		sys cp $pwmsBin $dir
	}
}

#-------------------------------------------------------------------------------
# Create 'software vesion' file
#-------------------------------------------------------------------------------
string createVersionFile(string htmlDir) {
	verFile := "$htmlDir/versions.txt"
	println "Creating versions file: '$verFile'"
	snpEffVer  := sys $snpeff -version 2>&1
	verFile.write(snpEffVer.trim().toUpper() + "\t$snpeffDownload\n");
	return verFile
}

#-------------------------------------------------------------------------------
# Human genomes data dirs
#-------------------------------------------------------------------------------
string[] dirsHg() {
	return dirData.dir("GRCh*") + dirData.dir("hg*")
}

#-------------------------------------------------------------------------------
# Download all datasets
#-------------------------------------------------------------------------------
void downloadAll() {
	dsets := ['ensembl', 'ensemblBfmpp', 'ucsc', 'ucsckg', 'mane', 'pdb', 'dbsnp', 'dbnsfp', 'cytobands', 'jaspar', 'gwasCatalog', 'nextprot', 'clinvar', 'flybase']
	for(string dset: dsets) dset.downloadOneSet()
}

#-------------------------------------------------------------------------------
# Download one dataset
#-------------------------------------------------------------------------------
void downloadOneSet(string setToDownload) {
    println "Download set '$setToDownload'"
	switch( setToDownload ) {
		case 'clinvar':
			downloadClinvar()
			break

		case 'dbnsfp':
			downloadDbNsfp()
			break

		case 'dbsnp':
			downloadDbSnp()
			break

		case 'ensembl':
			for( int ensemblRelease : ensemblReleases ) downloadEnsemblBulk(ensemblRelease)
			break

		case 'ensemblBfmpp':
			downloadEnsemblBfmpp()
			break

		case 'flybase':
			downloadFlyBase()
			break

		case 'gwasCatalog':
			downloadGwasCatalog()
			break

		case 'jaspar':
			downloadJaspar()
			break

        case 'mane':
            downloadManeGenome()
            break

		case 'nextprot':
			downloadNextProt()
			break

		case 'pdb':
			downloadPdb()
			break

		case 'ucsc':
			for( string ref : ucscGenomes ) {
				dg := new DownloadUcsc(ref)
				dg.download()
			}
			break

		case 'ucsckg':
			for( string ref : ucscGenomesKg ) {
				dg := new DownloadUcscKg(ref)
				dg.download()
			}
			break

		default:
			error("Unknown download set '$setToDownload'")
	}
}

#-------------------------------------------------------------------------------
# Get Azure query string from azure keys file, if not already defined
#-------------------------------------------------------------------------------
string getAzureQueryString(string type) {
	if( !azureQueryString.isEmpty() ) return azureQueryString
	println "Reading keys from '$azureKeysFile'"
	azureKeys := azureKeysFile.config()
	return azureKeys{"$type\_query_string"}
}

#-------------------------------------------------------------------------------
# Inger coordinates type fr huma genome: 'GRCh37' or 'GRCh38'
#-------------------------------------------------------------------------------
string grch(string hg) {
	if( hg.startsWith('GRCh37') ) return 'GRCh37'
	if( hg.startsWith('hg19') ) return 'GRCh37'
	if( hg.startsWith('GRCh38') ) return 'GRCh38'
	if( hg.startsWith('hg38') ) return 'GRCh38'
	error("Unknown coordinates for genome version '$hg'")
}

#-------------------------------------------------------------------------------
# Create distribution files
#-------------------------------------------------------------------------------
void makeDistro() {
	dirZip.mkdir()	# Make sure zip dir exists
	zipCore()
	zipDbs()
	# zipBundles()	# As of 2020-08 we'll stop using "bundles" because these were only used to reduce the number of files on SourceForge
}

#-------------------------------------------------------------------------------
# Run test cases
#-------------------------------------------------------------------------------
void runTests() {
	runTestsSnpSiftAll()
	runTestsSnpEffUnity()
	runTestsSnpEffIntegration()
}

#-------------------------------------------------------------------------------
# Run test cases: SnpSift
#-------------------------------------------------------------------------------
void runTestsSnpSiftAll() {
	task cd $HOME/workspace/SnpSift/ ; \
		java -Xmx4g \
			-cp $dirSnpEff/SnpSift.jar \
			org.junit.runner.JUnitCore \
			org.snpsift.testCases.TestSuiteAll \
			2>&1 \
			| tee $dirSnpEff/testcases.snpsift.all.txt
}

#-------------------------------------------------------------------------------
# Run test cases: SnpEff-Integration
#-------------------------------------------------------------------------------
void runTestsSnpEffIntegration() {
	task cd $HOME/workspace/SnpEff/ ; \
		java -Xmx4g \
			-cp $dirSnpEff/snpEff.jar \
			org.junit.runner.JUnitCore \
			org.snpeff.snpEffect.testCases.TestSuiteIntegration \
			2>&1 \
			| tee $dirSnpEff/testcases.snpeff.integration.txt
}

#-------------------------------------------------------------------------------
# Run test cases: SnpEff Unity
#-------------------------------------------------------------------------------
void runTestsSnpEffUnity() {
	# Run SnpEff test cases: Unity
	task cd $HOME/workspace/SnpEff/ ; \
		java -Xmx4g \
			-cp $dirSnpEff/snpEff.jar \
			org.junit.runner.JUnitCore \
			org.snpeff.snpEffect.testCases.TestSuiteUnity \
			2>&1 \
			| tee $dirSnpEff/testcases.snpeff.unity.txt
}

#-------------------------------------------------------------------------------
# Upload core files to Azure
#-------------------------------------------------------------------------------
void uploadCore2Azure() {
	aqs := getAzureQueryString('versions')
	zip := "snpEff_v" + versionSubUnd() + "_core.zip"
	zipLatest := "snpEff_latest_core.zip"
	azureContainer = 'versions'
	sys azcopy cp $zip       "$azureBlobEndpoint/$azureContainer/$zip$aqs"
	sys azcopy cp $zipLatest "$azureBlobEndpoint/$azureContainer/$zipLatest$aqs"
	# Add version file
	htmlDir := "$HOME/workspace/SnpEff/docs"
	verFile := htmlDir.createVersionFile()
	verBase := verFile.baseName()
	sys azcopy cp $verFile "$azureBlobEndpoint/$azureContainer/$verBase$aqs"
}

#-------------------------------------------------------------------------------
# Upload core files to SourceForge
#-------------------------------------------------------------------------------
void uploadCore2Sf() {
	zip := "snpEff_v" + versionSubUnd() + "_core.zip"
	zipLatest := "snpEff_latest_core.zip"
	sys scp -v $zip $zipLatest pcingola,snpeff@frs.sourceforge.net:/home/frs/project/s/sn/snpeff/
}

#-------------------------------------------------------------------------------
# Upload dbNSFP to Azure
#-------------------------------------------------------------------------------
void uploadDbNsfp2Azure() {
	error "Unimplamented!"
}

#-------------------------------------------------------------------------------
# Upload database files to Azure
#-------------------------------------------------------------------------------
void uploadDbs2Azure() {
	aqs := getAzureQueryString('databases')
	ver := 'v' + versionUnd()
	sys ln -s $dirZip $ver || true
	sys azcopy cp $ver/ "$azureBlobEndpoint/$azureContainer/$aqs" --recursive
}

#-------------------------------------------------------------------------------
# Upload database files to SourceForge
# DEPRECATED
#-------------------------------------------------------------------------------
void uploadDbs2Sf() {
	ver := versionUnd()
	sys scp -v $dirZip/snpEff_v$ver*.zip pcingola,snpeff@frs.sourceforge.net:/home/frs/project/s/sn/snpeff/databases/v$ver/
}

#-------------------------------------------------------------------------------
# Upload core development files to azure
#-------------------------------------------------------------------------------
void uploadDev2Azure() {
	aqs := getAzureQueryString('versions')
	azureContainer = 'versions'
	zipDev := "snpEff_development.zip"
	sys azcopy cp $zipDev "$azureBlobEndpoint/$azureContainer/$zipDev$aqs"
}

#-------------------------------------------------------------------------------
# SnpEff's version 
#-------------------------------------------------------------------------------
string version() {
	if( snpEffVersion == '' ) {
		snpEffVersion = sys $snpeff -version | cut -f 2
	}
	return snpEffVersion;
}

#-------------------------------------------------------------------------------
# SnpEff's version using underscores instead of '.'
#-------------------------------------------------------------------------------
string versionUnd() {
	ver := version()
	return ver.replace("\.","_").replaceAll("[a-zA-Z]", "").trim() # Remove sub-version letter
}

#-------------------------------------------------------------------------------
# SnpEff's version using underscores instead of '.'
#-------------------------------------------------------------------------------
string versionSubUnd() {
	ver := version()
	return ver.replace("\.","_").trim()
}

#-------------------------------------------------------------------------------
# Create bundle's ZIP files
#-------------------------------------------------------------------------------
void zipBundles() {
	println "Create bundles ZIP files"

	bundleList := "bundles.txt"
	bundleLines := sys cat $dirSnpEff/snpEff.config | grep .bundle. > $bundleList
	bundleList.rmOnExit()

	for( string bline : bundleList.read().lines() ) {
		bfields := bline.split(':')

		# Bundle name
		bname := bfields[0].trim().removeExt('.bundle')

		string[] bins
		for( string gen : bfields[1].split(' ') ) {
			bin := "$dirData/$gen/snpEffectPredictor.bin"
			if( bin.exists() ) bins.add( bin )
		}

		# Zip files
		zipList := bins.join(' ')
		zipName := "$dirZip/snpEff_v" + versionUnd() + "_" + bname + ".zip"
		if( zipName <- bins ) {
			println "\tZIP Bundle: $zipName"
			task zip $zipName $zipList
		}
	}
}

#-------------------------------------------------------------------------------
# Create SnpEff core Zip file
#-------------------------------------------------------------------------------
void zipCore() {
	ver := versionSubUnd()
	dir := "snpEff"
	sys rm -rvf $dir
	sys mkdir $dir

	# Copy core files
	sys cp -RvfL snpEff.config snpEff.jar SnpSift.jar LICENSE.md examples galaxy scripts exec $dir

	# Create 'core' zip file
	zip := "snpEff_v" + ver + "_core.zip"
	zipLatest := "snpEff_latest_core.zip"
	zipDev    := "snpEff_development.zip"
	sys rm -f $zip 2> /dev/null
	task { 
		sys zip -r $zip snpEff
		sys cp $zip $zipLatest
		sys cp $zip $zipDev
	}
}

#-------------------------------------------------------------------------------
# Create databases (non-bundled) Zip files
#-------------------------------------------------------------------------------
void zipDb(string gen) {
	bin := "data/$gen/snpEffectPredictor.bin"

	# Do not create ZIP file unless database exists
	if( bin.exists() ) {
		zip := "$dirZip/snpEff_v" + versionUnd() + "_$gen\.zip"
		println "\tZIP '$zip'"

		cyto := "data/$gen/cytoBand.txt.gz"
		if( ! cyto.exists() ) cyto = ""

		task( zip <- bin )	sys zip -r $zip data/$gen/*.bin $cyto
	} else {
		println "\tERROR: database $bin not found"
	}
}

#-------------------------------------------------------------------------------
# Create databases (non-bundled) Zip files
#-------------------------------------------------------------------------------
void zipDbs(string[] gens) {
	for( string gen : gens ) {
		zipDb(gen)
	}
}

#-------------------------------------------------------------------------------
# Create databases (non-bundled) Zip files
#-------------------------------------------------------------------------------
void zipDbs() {
	println "Create database ZIP files"
	dbList := "databases.txt"
	dbs := sys $snpeff databases | cut -f 1,4 > $dbList
	dbList.rmOnExit()

	for( string db : dbList.read().lines() ) {
		fields := db.split('\t')
		gen := fields[0].trim()
		bundle := fields[1].trim()

		# Do not zip bundled databases
		if( bundle.isEmpty() )	zipDb(gen)
	}
}

#-------------------------------------------------------------------------------
# Main
#-------------------------------------------------------------------------------

log("Building")

if( args.isEmpty() )			build()										# No arguments? Build (create JAR files)
if( bundleLists )				buildBundleLists()							# Create bundle list and config files
if( createConfig )				buildConfig()								# Create config file
if( createDocs )				buildDocs()									# Create documentation and web page
if( db || !dbs.isEmpty() )		buildDb()									# Build ALL genomes
if( dbTest )					buildDbTest()								# Build databases used in test cases
if( distro )					makeDistro()								# Create distribution files
if( distroCore )				zipCore()									# Create distribution files (core 'zip')
if( distroBundles )				zipBundles()								# Create 'bundle' distribution files
if( download )					downloadAll()								# Perform downloads
if( !downloadEnsembl.isEmpty() )	downloadEnsembl.downloadEnsemblGenome(downloadEnsemblSpecies)		# Download a specific ENSEMBL genome
if( downloadMane )				downloadManeGenome()						# Download MANE genome
if( !downloadNcbi.isEmpty() )	downloadNcbi.downloadNcbiGenome(ncbiId)		# Download NCBI genome
if( !downloadUcsc.isEmpty() )	downloadUcsc.downloadUcscGenome()			# Download UCSC genome (either refseq or kg)
if( !downloadSet.isEmpty() )	downloadSet.downloadOneSet()				# Download one set
if( make )						build()										# Build (create JAR files)
if( test )						runTests()									# Run test cases 
if( uploadCore )				uploadCore2Azure()							# Upload 'core' files
if( uploadDbs )					uploadDbs2Azure()							# Upload all database files in 'zip' dir
if( uploadDbNsfp )				uploadDbNsfp2Azure()						# Upload dbNsfp database files 
if( uploadDev )					uploadDev2Azure()							# Upload 'development' version
if( !zipGenome.isEmpty() )		zipDbs(zipGenome)							# Zip only 'zipGenome' database

wait
log("Done")

