cronshctl

#!/bin/bash


set -e
enable kill
for libdir in "$(dirname "`readlink -f "$0"`")" /usr/lib/cronsh /usr/share/cronsh ""
do
	if [ -z "$libdir" ]
	then
		echo "can not source cronsh-common" >&2
		false
	fi
	if [ -e "$libdir/cronsh-common" ]
	then
		. "$libdir/cronsh-common"
		break
	fi
done
set +e



find_last_attempt_number()
{
	cat "$1/latest-attempt-num" 2>/dev/null
}
find_last_attempt_pidfile()
{
	local lastattempt=`find_last_attempt_number "$1"`
	if [ -n "$lastattempt" ]
	then
		echo "$1/attempt-$lastattempt/pid"
	fi
}
find_current_attempt_pid()
{
	local lastattempt=`find_last_attempt_number "$1"`
	if [ ! -e "$1/attempt-$lastattempt/status" ]
	then
		local pidfile=$1/attempt-$lastattempt/pid
		if is_locked "$pidfile"
		then
			cat "$pidfile" 2>/dev/null
		fi
	fi
}

does_look_like_a_jobdir()
{
	[ -e "$1/lock" -a -e "$1/job" -a -d "$1/$CRONSH_WORKDIR_BASE/instance" ]
}


shopt -s extglob
shopt -s nullglob

cmd=$1
shift

case "$cmd" in
ps|history)
	usage()
	{
		echo "Usage: cronshctl ps [<options>] [<job-directory>]
Options:
 --last, -l [<N>]
 --successful, -s
 --failed, -f
 --all, -a
 --work-directory" >&2
	}
	
	show_last_n=''
	show_active_only=yes
	show_by_exitcode=''
	show_workdir=''
	declare -a show_instance_ids
	
	if [ "$cmd" = history ]
	then
		show_active_only=no
	fi
	
	while [ -n "$1" ]
	do
		case "$1" in
		-l|--last)
			show_active_only=no
			if [ "$2" -ge 0 ] 2>/dev/null
			then
				show_last_n=$2
				shift
			else
				show_last_n=1
			fi
			;;
		-s|--successful)
			show_active_only=no
			show_by_exitcode_grep_flag=''
			show_by_exitcode=yes
			;;
		-f|--failed)
			show_active_only=no
			show_by_exitcode_grep_flag='-v'
			show_by_exitcode=yes
			;;
		-a|--all)
			show_active_only=no
			;;
		--work-directory)
			show_workdir=yes
			;;
		-h|--help)
			usage
			exit 0
			;;
		--)	shift; break;;
		-*)	die "unknown option: $1";;
		*)	break;;
		esac
		shift
	done
	
	if [ $# = 0 ]
	then
		CRONSH_JOBDIR=.
	else
		CRONSH_JOBDIR=$1
		shift
	fi
	
	[ -d "$CRONSH_JOBDIR" ] || { usage; exit 2; }
	CRONSH_JOBDIR_ABS=`mk_abs_path "$CRONSH_JOBDIR"`
	CRONSH_STOPOLD_LOCKFILE=`get_stopold_lockfile_name`
	
	if [ "$show_active_only" = yes ]
	then
		show_instance_ids=($(gather_non_terminated_instance_ids "$CRONSH_JOBDIR"))
	else
		show_instance_ids=($(gather_all_instance_ids "$CRONSH_JOBDIR"))
	fi
	
	if [ "$show_by_exitcode" = yes ]
	then
		show_instance_ids=(`join -j1 \
			<(IFS=$'\n'; echo "${show_instance_ids[*]}") \
			<(grep $show_by_exitcode_grep_flag ' 0$' "$CRONSH_JOBDIR/status-history" | cut -f1 -d' ' | sort)
		`)
	fi
	
	if [ -n "$show_last_n" ] && [ $show_last_n -lt ${#show_instance_ids[@]} ]
	then
		show_instance_ids_=("${show_instance_ids[@]}")
		show_instance_ids=()
		for n in `seq $show_last_n -1 1`
		do
			show_instance_ids+=("${show_instance_ids_[-n]}")
		done
		unset show_instance_ids_
	fi
	
	
	# print header
	fmt="%-5s\t%5s\t%-${human_date_fmt_ls_width}s\t%-${human_date_fmt_ls_width}s\t%3s\t%5s\t%1s${show_workdir:+\t%s}\n"
	printf "$fmt" STATE PID STARTED ENDED "LAST STATUS" "CURRENT ATTEMPT PID" ATTEMPTS ${show_workdir:+WORKDIR}
	
	
	# list started (and maybe terminated) instances
	for instance_id in "${show_instance_ids[@]}"
	do
		instance_started_nsec=`get_instance_start_nsec $instance_id`
		instancedir=`get_fq_instance_dir "$CRONSH_JOBDIR" $instance_id`
		
		if is_locked "$instancedir/pid"
		then
			instance_state=RUN
			instancepid=`cat "$instancedir/pid" 2>/dev/null`
		else
			instance_state=TERM
			if [ "$show_active_only" = yes ]
			then
				continue
			fi
			instancepid=''
		fi
		
		started=`date +"$human_date_fmt_ls" -d @$instance_started_nsec`
		ended=`date +"$human_date_fmt_ls" -d "@$(cat "$instancedir/end" 2>/dev/null)" 2>/dev/null`
		attempts=`find_last_attempt_number "$instancedir"`
		
		if [ -z "$attempts" ]
		then
			laststatus=''
			attemptpid=''
			attempts=0
		else
			if [ -e "$instancedir/attempt-$attempts/status" ]
			then
				# last job attempt is ended
				lastattempt=$attempts
				attemptpid=''
			else
				# latest job attempt is in progress
				lastattempt=$[attempts - 1]
				pidfile=$instancedir/attempt-$attempts/pid
				if is_locked "$pidfile"
				then
					attemptpid=`cat "$pidfile"`
				else
					attemptpid=''
				fi
			fi
			laststatus=`cat "$instancedir/attempt-$lastattempt/status" 2>/dev/null`
			# TODO: get final status code from status-history file for TERM instances
		fi
		
		printf "$fmt" "$instance_state" "$instancepid" "$started" "$ended" "$laststatus" "$attemptpid" "$attempts" ${show_workdir:+"$instancedir"}
	done
	
	if [ "$show_by_exitcode" != yes ]
	then
		# list queued instances
		for pending_lock_pid in `list_pending_locks_on_file "$CRONSH_JOBDIR/lock" | get_pids_from_locks_list`
		do
			instance_state=QUEUE
			# cronsh process is the parent of flock process which is pending on lock
			instancepid=`get_parent_pid "$pending_lock_pid"`
			instance_started_datestr=`ps o lstart= -p "$instancepid"`
			started=`date +"$human_date_fmt_ls" -d "$instance_started_datestr"`
			ended=''
			laststatus=''
			attemptpid=''
			attempts=''
			
			printf "$fmt" "$instance_state" "$instancepid" "$started" "$ended" "$laststatus" "$attemptpid" "$attempts" ${show_workdir:+""}
		done
		
		# list the instance (if any) which has not started yet but trying to stop its predecessor
		if is_locked "$CRONSH_STOPOLD_LOCKFILE"
		then
			instance_state=INIT
			instancepid=`cat "$CRONSH_STOPOLD_LOCKFILE"`
			instance_started_nsec=`stat -c %Y "$CRONSH_STOPOLD_LOCKFILE"`
			started=`date +"$human_date_fmt_ls" -d @$instance_started_nsec`
			ended=''
			laststatus=''
			attemptpid=''
			attempts=''
			
			printf "$fmt" "$instance_state" "$instancepid" "$started" "$ended" "$laststatus" "$attemptpid" "$attempts" ${show_workdir:+""}
		fi
	fi
	;;

log)
	usage()
	{
		echo "Usage: cronshctl log [-f] <job-directory> [...]
Options:
 -f   follow logs (see: tail -F)" >&2
	}
	
	logfiles=()
	declare -A instance_pids=()
	tail_follow=no
	nologfileyet_reported=no
	
	while [ -n "$1" ]
	do
		case "$1" in
		-f)
			tail_follow=yes
			;;
		-h|--help)
			usage
			exit 0
			;;
		--)	shift; break;;
		-*)	die "unknown option: $1";;
		*)	break;;
		esac
		shift
	done
	
	if [ $# = 0 ]
	then
		usage
		exit 2
	else
		args=("$@")
	fi
	
	while true
	do
		for jobdir in "${args[@]}"
		do
			[ -d "$jobdir" ] || { usage; exit 2; }
			
			for instance_id in `gather_running_instance_ids "$jobdir"`
			do
				instance_dir=`get_fq_instance_dir "$jobdir" "$instance_id"`
				instance_pids[`cat "$instance_dir"/pid`]=1
				logfiles+=("$instance_dir"/std{err,out}{,-job}.txt)
			done
		done
		
		if [ ${#logfiles[@]} = 0 ]
		then
			if [ $tail_follow = yes ]
			then
				if [ $nologfileyet_reported = no ]
				then
					note "no log file yet"
					nologfileyet_reported=yes
				fi
				sleep 1
			else
				note "no log file"
				exit
			fi
		else
			break
		fi
	done
	
	if [ $tail_follow = yes ]
	then
		# create a subprocess which does nothing but live until
		# all of the job instances are running to let tail return
		# when no more log update can happen.
		parent_pid=$$
		{
			while kill -0 $parent_pid 2>/dev/null
			do
				# wait while any instance is running
				for p in "${!instance_pids[@]}"
				do
					if ! kill -0 "$p" 2>/dev/null
					then
						unset "instance_pids[$p]"
					fi
				done
				if [ ${#instance_pids[@]} = 0 ]
				then
					exit
				fi
				sleep 1
			done
		} &
	fi
	
	tail_opts=(-n +1)
	if [ $tail_follow = yes ]
	then
		tail_opts+=(--pid=$! -F)
	fi
	
	tail "${tail_opts[@]}" "${logfiles[@]}"
	;;

attach)
	usage()
	{
		echo "Usage: cronshctl attach [-r] <job-instance-directory>
Options:
 -r   readonly mode" >&2
	}
	
	sock=$1/console
	
	if [ ! -S "$sock" ] && does_look_like_a_jobdir "$1"
	then
		note "Attaching to most recent started job instance..."
		sock=$1/$CRONSH_WORKDIR_BASE/most-recent-started/console
	fi
	
	keept ba "$sock"
	;;
stop)
	usage()
	{
		echo "Usage: cronshctl stop [-g | --pgroup] [<job-directory> | <job-instance-directory>] [...]" >&2
	}
	
	terminate_pgroup=no
	check_interval=1
	pgroup_signal_sent=no
	pgroup_kill_delay=2
	instance_workdirs=()
	active_instance_workdirs=()
	declare -A pending_instances=()
	
	while [ -n "$1" ]
	do
		case "$1" in
		-g|--pgroup)
			terminate_pgroup=yes
			;;
		-h|--help)
			usage
			exit 0
			;;
		--)	shift; break;;
		-*)	die "unknown option: $1";;
		*)	break;;
		esac
		shift
	done
	
	if [ $# = 0 ]
	then
		usage
		exit 2
	fi
	
	while [ -n "$1" ]
	do
		[ -d "$1" ] || { usage; exit 2; }
		
		if [ -e "$1/lock" ]
		then
			# If there is a file called "lock" in the directory, we count it as a Job Root dir
			# and add all instance directories to the list, as it'll be filtered later;
			jobdir=$1
			echo "$jobdir: searching instance dirs..." >&2
			for instance_id in `gather_non_terminated_instance_ids "$jobdir"`
			do
				instance_workdirs+=(`get_fq_instance_dir "$jobdir" "$instance_id"`)
			done
		elif [ -e "$1/pid" ]
		then
			# Otherwise it's must be a Job Instance dir
			instance_workdirs+=("$1")
		else
			echo "$1: does not seem neither a job dir nor an instance dir" >&2
			exit 3
		fi
		shift
	done
	
	
	# Signal Job Instance processes
	for instancedir in "${instance_workdirs[@]}"
	do
		# Filter for active Job Instances only
		if is_locked "$instancedir/pid"
		then
			# First inform Job Instance process about termination
			instancepid=`cat "$instancedir/pid"`
			# Normally Job Instance process waits until Job Attempt exits.
			# So it's okay here to assume Instance process is running under this pid.
			# TODO: take into account when Instance exits before Job Attempt.
			kill -s TERM $instancepid
			pending_instances[$instancedir]=$instancepid
			active_instance_workdirs+=("$instancedir")
		fi
	done
	signal_time=`date +%s`
	
	
	while true
	do
		for instancedir in "${!pending_instances[@]}"
		do
			if ! is_locked "$instancedir/pid"
			then
				unset "pending_instances[$instancedir]"
			fi
		done
		
		if [ ${#pending_instances[@]} = 0 ]
		then
			break
		fi
		
		echo "Wait for termination (${#pending_instances[@]}/${#active_instance_workdirs[@]})" >&2
		
		if [ $terminate_pgroup = yes -a $pgroup_signal_sent = no ]
		then
			if [ $[signal_time + pgroup_kill_delay] -le `date +%s` ]
			then
				echo "Terminating Job Process Group(s)" >&2
				
				for instancedir in "${!pending_instances[@]}"
				do
					attemptpidfile=`find_last_attempt_pidfile "$instancedir"`
					if is_locked "$attemptpidfile"
					then
						attemptpid=`cat "$attemptpidfile"`
						if [ -n "$attemptpid" ]
						then
							kill -s TERM -- -$attemptpid
						fi
					fi
				done
				
				pgroup_signal_sent=yes
			fi
		fi
		
		sleep $check_interval
	done
	;;

*)
	echo "cronshctl [ps | stop | log | attach | history]" >&2
	exit 1
	;;
esac