#!/bin/bash
#* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *#
#*                                                                           *#
#*                  This file is part of the class library                   *#
#*       SoPlex --- the Sequential object-oriented simPlex.                  *#
#*                                                                           *#
#*  Copyright (c) 1996-2025 Zuse Institute Berlin (ZIB)                      *#
#*                                                                           *#
#*  Licensed under the Apache License, Version 2.0 (the "License");          *#
#*  you may not use this file except in compliance with the License.         *#
#*  You may obtain a copy of the License at                                  *#
#*                                                                           *#
#*      http://www.apache.org/licenses/LICENSE-2.0                           *#
#*                                                                           *#
#*  Unless required by applicable law or agreed to in writing, software      *#
#*  distributed under the License is distributed on an "AS IS" BASIS,        *#
#*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *#
#*  See the License for the specific language governing permissions and      *#
#*  limitations under the License.                                           *#
#*                                                                           *#
#*  You should have received a copy of the Apache-2.0 license                *#
#*  along with SoPlex; see the file LICENSE. If not email soplex@zib.de.     *#
#*                                                                           *#
#* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *#
#
# Call with "make testcluster"
#
# The queue is passed via ${QUEUE} (possibly defined in a local makefile in scip/make/local).
#
# For each run, we can specify the number of nodes reserved for a run via ${PPN}. If tests runs
# with valid time measurements should be executed, this number should be chosen in such a way
# that a job is run on a single computer, i.e., in general, ${PPN} should equal the number of cores
# of each computer. Of course, the value depends on the specific computer/queue.
#
# To get the result files call in directory check/
# ./evalcheck_cluster.sh ${OUTPUTDIR}/check.${TSTNAME}.${BINID}.${SETNAME}.eval
# This leads to result files
#  - ${OUTPUTDIR}/check.${TSTNAME}.${BINID}.${SETNAME}.out
#  - ${OUTPUTDIR}/check.${TSTNAME}.${BINID}.${SETNAME}.res
#  - ${OUTPUTDIR}/check.${TSTNAME}.${BINID}.${SETNAME}.err

TSTNAME="${1}"
EXECUTABLE="${2}"
BINID="${3}"
SETTINGS="${4}"
TIMELIMIT="${5}"
MEMLIMIT="${6}"
CONTINUE="${7}"
QUEUETYPE="${8}"
QUEUE="${9}"
PPN="${10}"
CLIENTTMPDIR="${11}"
NOWAITCLUSTER="${12}"
EXCLUSIVE="${13}"
OUTPUTDIR="${14}"
SEEDS="${15}"

# check if all variables defined (by checking the last one)
if test -z "${OUTPUTDIR}"
then
    echo Skipping test since not all variables are defined
    echo "TSTNAME       = ${TSTNAME}"
    echo "EXECUTABLE    = ${EXECUTABLE}"
    echo "BINID         = ${BINID}"
    echo "SETTINGS      = ${SETTINGS}"
    echo "TIMELIMIT     = ${TIMELIMIT}"
    echo "MEMLIMIT      = ${MEMLIMIT}"
    echo "CONTINUE      = ${CONTINUE}"
    echo "QUEUETYPE     = ${QUEUETYPE}"
    echo "QUEUE         = ${QUEUE}"
    echo "PPN           = ${PPN}"
    echo "CLIENTTMPDIR  = ${CLIENTTMPDIR}"
    echo "NOWAITCLUSTER = ${NOWAITCLUSTER}"
    echo "EXCLUSIVE     = ${EXCLUSIVE}"
    echo "OUTPUTDIR     = ${OUTPUTDIR}"
    echo "SEEDS         = ${SEEDS}"
    exit 1
fi

# call routines for creating the result directory, checking for existence
# of passed settings, etc
# defines the following environment variables: SOPLEXPATH, EXECUTABLE, FULLTSTNAME, SOLUFILE, SETTINGSFILE
. ./configuration_set.sh

# create output directory
mkdir -p "${SOPLEXPATH}/${OUTPUTDIR}"

# check if binary exists
if test ! -e "${EXECUTABLE}"
then
    echo "Skipping test since the binary ${EXECUTABLE} does not exist."
    exit 1
fi

# configure cluster-related environment variables
# defines the following environment variables: NICE, ACCOUNT, CLUSTERQUEUE, HARDTIMELIMIT
. ./configuration_cluster.sh

# run different random seeds
for (( s=0; s<=${SEEDS}; s++ ))
do
  FILENAME="${SOPLEXPATH}/${OUTPUTDIR}/check.${TSTNAME}.${BINID}.${QUEUE}.${SETTINGS}"
  if (( SEEDS > 0 )); then
      FILENAME="${FILENAME}-s${s}"
  fi
  EVALFILE="${FILENAME}.eval"
  echo > "${EVALFILE}"

  # counter to define file names for a test set uniquely
  COUNT=0

  # iterate over instances in .test file
  for i in $(cat "${FULLTSTNAME}")
  do
      if test "${i}" = "DONE"
          then
          break
      fi

      # increase the index for the inctance tried to solve, even if the filename does not exist
      COUNT=$((COUNT + 1))

      # check if problem instance exists
      if test -f "${SOPLEXPATH}/${i}"
      then

          # the cluster queue has an upper bound of 2000 jobs; if this limit is
          # reached the submitted jobs are dumped; to avoid that we check the total
          # load of the cluster and wait until it is save (total load not more than
          # 1900 jobs) to submit the next job.
          if test "${NOWAITCLUSTER}" != "1"
          then
              if test  "${QUEUETYPE}" != "qsub"
              then
                  echo "waitcluster does not work on slurm cluster"
              fi
              ./waitcluster.sh 1600 "${QUEUE}" 200
          fi

          SHORTFILENAME=$(basename "${i}" .gz)
          SHORTFILENAME=$(basename "${SHORTFILENAME}" .mps)
          SHORTFILENAME=$(basename "${SHORTFILENAME}" .lp)
          SHORTFILENAME=$(basename "${SHORTFILENAME}" .opb)

          FILENAME="${USER}.${TSTNAME}.${COUNT}_${SHORTFILENAME}.${BINID}.${QUEUE}.${SETTINGS}"
          if (( SEEDS > 0 )); then
              FILENAME="${FILENAME}-s${s}"
          fi
          BASENAME="${SOPLEXPATH}/${OUTPUTDIR}/${FILENAME}"

          TMPFILE="${BASENAME}.tmp"
          SETFILE="${BASENAME}.set"

          echo "${BASENAME}" >> "${EVALFILE}"

          # in case we want to continue we check if the job was already performed
          if test "${CONTINUE}" != "false"
          then
              if test -e "${OUTPUTDIR}/${FILENAME}.out"
              then
                  echo "skipping file ${i} due to existing output file ${FILENAME}.out"
                  continue
              fi
          fi

          # additional environment variables needed by runcluster.sh
          export SOLVERPATH="${SOPLEXPATH}"
          export EXECNAME="${EXECUTABLE}"
          export BASENAME="${FILENAME}" # careful, runcluster.sh uses BASENAME in a different sense
          export FILENAME="${i}"        # careful, runcluster.sh uses FILENAME in a different sense
          export OUTPUTDIR="${OUTPUTDIR}"
          export TIMELIMIT="${TIMELIMIT}"
          export SETTINGS="${SETTINGS}"
          export INSTANCE="${SOPLEXPATH}/${i}"
          export CLIENTTMPDIR="${CLIENTTMPDIR}"

          # Create testset, without printing output
          "${EXECNAME}" --loadset="${SETTINGSFILE}" -t"${TIMELIMIT}" --saveset="${SOLVERPATH}/${OUTPUTDIR}/${BASENAME}.set" > /dev/null

          if test  "${QUEUETYPE}" = "srun"
          then
              sbatch --job-name=SPX-"${SHORTFILENAME}" --constraint="${CONSTRAINT}" --mem="${HARDMEMLIMIT}" -p "${CLUSTERQUEUE}" -A "${ACCOUNT}" --time="${HARDTIMELIMIT}" ${NICE} ${EXCLUSIVE} --output=/dev/null runcluster.sh
          else
              # -V to copy all environment variables
              qsub -l walltime="${HARDTIMELIMIT}" -l mem="${HARDMEMLIMIT}" -l nodes=1:ppn="${PPN}" -N SOPLEX"${SHORTFILENAME}" -V -q "${CLUSTERQUEUE}" -o /dev/null -e /dev/null runcluster.sh
          fi
      else
          echo "input file ${SOPLEXPATH}/${i} not found!"
      fi
  done
done
