#!/bin/bash
# 31-08-2017	lines 32-41	| added a disk quota check to avoid hitting the limit
# 01-09-2017	lines 96-100 	| fixed a bug causing the script to try to subint more trajs than the one in DynamicsINPUTS

# PARAMETERS ##############################################################################################

interval=1 	# define interval (hours) after which you want the script to wake up

FreeNodes=0	# define the amount of nodes you want to leave free for your colleagues 
		# this number might depend on the amount of cake your colleagues are willing to give you

############################################################################################################

# determine how many nodes the machine has 
TotNodes=`pbsnodes -l all | wc -l` 

# allocate half of the total number of nodes for calculations
# note that this is in python because bash sucks and I don't trust it with numbers
MaxJobN=`python -c "tot = float(${TotNodes}) / 2. + 10; print int(tot)"`  


# write header
echo "### AIMD SUBMITTER ####################################"	>> .Submitter.log
echo "# Initiated on:      `date | awk '{print $2,$3,$4}'`"	>> .Submitter.log
echo "# Working Directory: `pwd`"				>> .Submitter.log
echo "# Updating interval: ${interval} hour(s)"			>> .Submitter.log
echo "#######################################################" 	>> .Submitter.log

# start infinite loop
while [ 1 ]
do
	# check if you are close to the disk quota ##############################
	#########################################################################
	read BLOCKS QUOTA <<< `quota -s | tail -n1 | awk '{print $2,$3}' | sed 's/M/ /g'`
	free=`python -c "space = int(${QUOTA}) - int(${BLOCKS}); print space "`
	if [ "${free}" -le "1000" ]
	then
		echo "##### disk free space low and very close to quota: free ${free}M"	>> .Submitter.log
		echo "##### submission suspended on `date | awk '{print $2,$3,$4}'`"	>> .Submitter.log
		exit 0
	fi

	# check if you are done #################################################
	#########################################################################

	# check the number of completed trajectories
	comp=`./Traj_Analysis.py | grep completed | awk '{print $NF}'`
	# since it may start with some zeroes this tells the script to read in base 10 and not in base 8
	completed=`python -c "print int('${comp}',10)"`

	# tot number of jobs to be submitted
	Nposcars=`ls DynamicsINPUTS/ | egrep "^[0-9]...$" | wc -l`

	if [ "${completed}" -eq "${Nposcars}" ]
	then
		echo "##### submission completed on `date | awk '{print $2,$3,$4}'`" >> .Submitter.log
		exit 0
	fi

	#########################################################################
	#########################################################################

	# check how many trajs are running for the current user
	Njobs=`qstat -u ${USER} | grep SRP | grep -v " C " | wc -l`

	# check the total number of free node
	AvailableNodes=`pbsnodes -l free | wc -l`
	
	#########################################################################
	#########################################################################

	# take care of restarts: check every folder...
	for folder in `ls | egrep "^[0-9]...$"`
	do
		if 	test -e ${folder}/outcome 		&& # if outcome file exists 
			grep -q UNCLEAR ${folder}/outcome 	&& # if the traj is unclear
			[ "${Njobs}" -lt "${MaxJobN}" ] 	&& # if the user can still submit jobs
			[ "${AvailableNodes}" -gt "${FreeNodes}" ] # if there are more than ${FreeNodes} available
		then
			echo "---> traj restarted: ${folder} on `date | awk '{print $2,$3,$4}'`" >> .Submitter.log

			# restarting 
			./restart.py ${folder} &>/dev/null

			# update running jobs number
	                Njobs=`qstat -u ${USER} | grep SRP | grep -v " C " | wc -l`

			# update the total number of free nodes
			let AvailableNodes--
		fi
	done

	#########################################################################
	#########################################################################

	# check how many jobs have been submitted already by counting the folders
	Ndone=`ls | egrep "^[0-9]...$" | wc -l`

	# submit more trajs until there are ${MaxJobN} jobs running
	# but leave at least ${FreeNodes} nodes empty (be nice with your colleagues!!)
	while [ "${Njobs}" -lt "${MaxJobN}" ] && [ "${AvailableNodes}" -gt "${FreeNodes}" ] && [ "${Ndone}" -lt "${Nposcars}" ]
	do
		let Ndone++	

		# POSCAR number to be submitted
		printf -v to_be_sub "%04d" ${Ndone}
	
		# preparing calculation	
		mkdir ${to_be_sub}
		cp DynamicsINPUTS/${to_be_sub}/POSCAR ${to_be_sub}/.
	
		echo "---> traj submitted: ${to_be_sub} on `date | awk '{print $2,$3,$4}'`" >> .Submitter.log
	
		# submitting
		./dynamics-sub.py &>/dev/null
	
		# update running jobs number
		Njobs=`qstat -u ${USER} | grep SRP | grep -v " C " | wc -l`	
	
		# update the total number of free nodes
	        let AvailableNodes--
	done
	
	#########################################################################
	#########################################################################

	# sleep ${interval} hours
	sleep ${interval}h

done