#!/bin/bash -e

#
# Author: Alfio Lazzaro, alfio.lazzaro@mat.ethz.ch (2013-2015)
# Script to generate LIBSMM library
# Run ./generate -h to see the help
#
echo
echo "Script to generate LIBSMM library."
echo "Author: Alfio Lazzaro, alfio.lazzaro@mat.ethz.ch (2013-2015)"
echo

#
# Source the library with all routines
#
source generate.bash

#
# Default OPTIONS values
#
def_config_dir="config"
def_config_file="${def_config_dir}/cray.gnu"
def_SIMD="avx"
def_ntasks="1"
def_base_work_dir="output"
def_target="all"
def_njobs="1"
def_wlm="slurm"
def_wtime="01:00:00"

show_help() {
    echo "Run with: ./generate [OPTIONS] [COMMAND]"
    echo
    echo "OPTIONS are:"
    echo "   -h               : show this help."
    echo "   -c <config file> : config file to use (declared inside ${def_config_dir} directory)."
    echo "                      Default value is \"${def_config_file}\"."
    echo "   -j <#>           : set the number of jobs for batch submission. Setting to zero means no batch submission."
    echo "                      Default value is \"${def_njobs}\"."
    echo "   -s <SIMD type>   : SIMD registers type (sse, avx, avx2, knc, avx512)."
    echo "                      Default value is \"${def_SIMD}\"."
    echo "   -t <#>           : set the number of tasks per each node."
    echo "                      Default value is ${def_ntasks}."
    echo "   -w <wlm type>    : workload manager for batch submission. The value must correspond to one of the files ${def_config_dir}/*.wlm."
    echo "                      Default value is \"${def_wlm}\"."
    echo "   -m <time>        : time limit for batch execution."
    echo "                      Default value is \"${def_wtime}\"."
    echo "   -a <target>      : Target for make compilation. Values are: "
    echo "                           all: generate, compile and run the kernels."
    echo "                        source: only generate the source file kernels."
    echo "                       compile: generate and compile the kernels."
    echo "                      Default value is \"${def_target}\"."
    echo
    echo "COMMAND is one of the followings:"
    echo "   tiny1  : it runs the tiny phase. Batch execution if requested."
    echo "   tiny2  : collect the results of the tiny phase."
    echo "            Run automatically during tiny1 when there is no batch execution."
    echo "   small1 : it runs the small phase. Batch execution if requested."
    echo "   small2 : collect the results of the small phase. "
    echo "            Run automatically during small1 when there is no batch execution."
    echo "   lib    : generate the library. Batch execution if requested (a single job needed)."
    echo "   check1 : run the checks on the library. Batch execution if requested."
    echo "   check2 : collect the results of the checks."
    echo "            Run automatically during check1 when there is no batch execution."
    echo "            NOTE: same number of jobs used in check1 must be used for check2."
    echo "NOTE: COMMANDs must be executed in the above order."
    echo
    echo "Special COMMAND:"
    echo "     clean : remove intermediate files (but not some key output and the library itself)."
    echo " realclean : remove all intermediate files."
    echo
    exit
}

#
# Load OPTIONS
#
check_number()
{
    if ! [[ "$1" =~ ^[0-9]+$ ]] ; then
	echo "Error: $1 is not a valid number for -$2 option!."
	echo "Run ./generate -h for help."
	exit
    fi

    # Check for positive numbers
    if [ $# -gt 2 -a $1 -le 0 ]; then
	echo "Error: $1 must be positive for -$2 option!."
	echo "Run ./generate -h for help."
	exit
    fi

}

while getopts "c:hj:s:t:w:m:a:" OPTION; do
    case $OPTION in
	c)
	    config_file=$OPTARG
	    ;;
	h)
	    show_help
	    ;;
	j)
	    jobs=$OPTARG
	    check_number $jobs $OPTION
	    ;;
	s)
	    SIMD=$OPTARG
	    ;;
	t)
	    ntasks=$OPTARG
	    check_number $ntasks $OPTION $ntasks
	    ;;
	w)
	    wlm=$OPTARG
	    ;;
	m)
	    wtime=$OPTARG
	    ;;
	a)
	    case $OPTARG in
		all|source|compile)
		    target=$OPTARG
	    	    ;;
		*)
		    echo "Warning: target \"$OPTARG\" unknown. Run ./generate -h for help."
		    exit
		    ;;
	    esac
	    ;;
        ?)
            exit
            ;;
    esac
done
shift $(( OPTIND - 1))

#
# Use default OPTIONS values if they were not declared
#
if [ -z "${config_file}" ]; then
    config_file=${def_config_file}
fi

if [ -z "${SIMD}" ]; then
    SIMD=${def_SIMD}
fi

if [ -z "${ntasks}" ]; then
    ntasks=${def_ntasks}
fi

if [ -z "${jobs}" ]; then
    jobs=${def_njobs}
fi

if [ -z "${wlm}" ]; then
    wlm=${def_wlm}
fi

if [ -z "${wtime}" ]; then
    wtime=${def_wtime}
fi

if [ -z "${target}" ]; then
    target=${def_target}
fi

base_work_dir=$def_base_work_dir

#
# Check COMMAND value
#
if [ $# -eq 0 ]; then
    echo "Missing COMMAND! Run ./generate -h for help."
    exit
fi

cmd=`echo $1 | awk '{ print tolower($0)}'`

case $cmd in
    tiny1|tiny2|small1|small2|lib|check1|check2)
	;;
    clean|realclean)
	echo "Remove files."
	rm -fR run_tiny_*/
	rm -fR run_small_*/
	rm -fR run_lib_*/
	rm -fR run_check_*/
	rm -f *.mod Makefile.* *.x *.o
	rm -f *~
	if [ "$cmd" == "realclean" ]; then
	    rm -f *.out
	    rm -fR lib
	fi
	echo
	exit
	;;
    *)
	echo "Unknown COMMAND \"$1\". Run ./generate -h for help."
	exit
	;;
esac

#
# Check OPTIONS values
#
if [ ! -e ${config_file} ]; then
    echo "Error: config file \"${config_file}\" doesn't exist!"
    echo "Available config files are:"
    ls ${def_config_dir} | grep -v "wlm"
    exit
fi

if [ ! -e ${def_config_dir}/${wlm}.wlm ]; then
    echo "Error: corresponding wlm file to \"${wlm}\" (${wlm}.wlm) doesn't exist inside ${def_config_dir} directory!"
    echo "Available wlm files are:"
    ls ${def_config_dir} | grep "wlm" | cut -d'.' -f1
    exit
fi

if [ "${cmd}" == "lib" -a ${jobs} -gt 0 ]; then
    jobs=1
fi

#
# Source the configuration files
#
source config.in
source ${config_file}

if [[ ( -n "${libxsmm_dir}") ]]; then
    echo "Use libxsmm from ${libxsmm_dir}"
    if [[ ( "${data_type}" != "1" ) && ( "${data_type}" != "2" ) ]]; then
	echo "Error: libxsmm doesn't support complex numbers."
	exit
    fi
    if [[ ( "${transpose_flavor}" != "1" ) ]]; then
	echo "Error: libxsmm supports only NN transposity."
	exit
    fi
fi

case `echo ${SIMD} | awk '{ print tolower($0)}'` in
    sse)
	SIMD_size=16
	SIMD_libxsmm="wsm"
	SIMD_libxsmm_target="SSE=3"
	;;
    avx)
	SIMD_size=32
	SIMD_libxsmm="snb"
	SIMD_libxsmm_target="AVX=1"
	;;
    avx2)
	SIMD_size=32
	SIMD_libxsmm="hsw"
	SIMD_libxsmm_target="AVX=2"
	;;
    knc)
	if [ ! -n "${target_compile_offload}" ]; then
            echo "Error: knc requires to set \`target_compile_offload' variable in the config file!"
	    exit
	fi
	SIMD_size=64
	SIMD_libxsmm="knc"
	SIMD_libxsmm_target="OFFLOAD=1"
	;;
    avx512)
	SIMD_size=64
	SIMD_libxsmm="knl"
	SIMD_libxsmm_target="AVX=3"
	;;
    *)
	echo "Error: SIMD register type \"${SIMD}\" doesn't exist!"
	echo "Run ./generate -h for help."
	exit
	;;
esac

dims_tiny=`echo ${dims_tiny} | tr "\n" " "` # remove \n
dims_small=`echo ${dims_small} | tr "\n" " "` # remove \n

#
# Dump configuration
#
echo "Config file                   : \"${config_file}\""
echo "Number of tasks per each node : ${ntasks}"
echo "Number of batch jobs          : ${jobs}"
echo "WLM                           : ${wlm}"
echo "SIMD register type            : ${SIMD}"
echo "Limit time batch              : ${wtime}"
echo "COMMAND                       : \"${cmd}\""
echo "Make target                   : \"${target}\""
echo

#
# customize target for libsmm
#
target+="_libsmm"

#
# Set working variables
#
config_file_name=`basename ${config_file}`
work_dir="${base_work_dir}_${config_file_name}"

if [ ${jobs} -gt 0 ]; then
    source ${def_config_dir}/${wlm}.wlm

    case $cmd in
	tiny1|small1|lib|check1)
	    run_cmd=batch_cmd
	    ;;
	tiny2|small2|check2)
	    run_cmd=true
	    ;;
    esac
else
    # Require 1 job
    jobs=1

    case $cmd in
	tiny2|small2|check2)
            run_cmd=true
            ;;
    esac
fi

type_label="_"
case "${data_type}" in
    1 )
	type_label+="d"
	gemm="DGEMM"
        strdat="REAL(KIND=KIND(0.0D0))"
	data_libxsmm="DP"
	;;
    2 )
	type_label+="s"
	gemm="SGEMM"
	strdat="REAL(KIND=KIND(0.0))"
	data_libxsmm="SP"
	;;
    3 )
	type_label+="z"
	gemm="ZGEMM"
	strdat="COMPLEX(KIND=KIND(0.0D0))"
	;;
    4 )
	type_label+="c"
	gemm="CGEMM"
	strdat="COMPLEX(KIND=KIND(0.0))"
	;;
esac
case "${transpose_flavor}" in
    1 )
	type_label+="nn"
	ta="N"
	tb="N"
	decl="A(M,K), B(K,N)"
	lds="LDA=M ; LDB=K"
	;;
    2 )
	type_label+="tn"
	ta="T"
	tb="N"
	decl="A(K,M), B(K,N)"
	lds="LDA=K ; LDB=K"
	;;
    3 )
	type_label+="nt"
	ta="N"
	tb="T"
	decl="A(M,K), B(N,K)"
	lds="LDA=M ; LDB=N"
	;;
    4 )
	type_label+="tt"
	ta="T"
	tb="T"
	decl="A(K,M), B(N,K)"
	lds="LDA=K ; LDB=N"
	;;
esac

run_dir="run"
make_file="Makefile."
case $cmd in
    tiny1|tiny2)
	run_dir+="_tiny"
	make_file+="tiny"
	;;
    small1|small2)
	run_dir+="_small"
	make_file+="small"
	;;
    lib)
	run_dir+="_lib"
	make_file+="lib"
	;;
    check1|check2)
	run_dir+="_check"

	;;
esac
run_dir+="${type_label}"
make_file+="${type_label}_${config_file_name}"

tiny_file="tiny_gen_optimal${type_label}_${config_file_name}.out"
small_file="small_gen_optimal${type_label}_${config_file_name}.out"
test_file="test_smm${type_label}"

library="smm${type_label}_${config_file_name}"
archive="../lib/lib${library}.a"

#
# Run the command!
#

mkdir -p ${run_dir}

case $cmd in
    tiny1|tiny2)
	do_generate_tiny
	if [ "$cmd" == "tiny1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with tiny2 command for collecting all results."
	    echo
	fi
	;;
    small1|small2)
	do_generate_small
	if [ "$cmd" == "small1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with small2 command for collecting all results."
	    echo
	fi
	;;
    lib)
	do_generate_lib
	echo
	echo "Remember to run the check* commands to check the correctness and performance of the library."
	echo
	;;
    check1|check2)
	do_check
	if [ "$cmd" == "check1" -a -n "${run_cmd}" ]; then
	    echo
	    echo "Wait for completion of all jobs, then run with check2 command for collecting all results."
	    echo
	fi
	;;
esac

