#!/bin/bash
#SBATCH -J sphere
#SBATCH --time=1:00:00
#SBATCH --mail-type=END
#SBATCH --mail-user=jiaxing.qi@uni-siegen.de
#SBATCH --exclusive
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=12
#SBATCH --partition=short
#SBATCH --mem-per-cpu=MaxMemPerCPU

# module load PrgEnv/intel-openmpi

export LEVEL=8
# number of iteration to run
# the iteration must be a factor of 64 for the multilevel mesh
export TMAX=512

# executables
# export musubi=~/exec/musubi
export musubi=~/apes/musubi/build/musubi
# export musubi=~/apes/musubi/build/debug/musubi
export seeder=~/apes/seeder/build/seeder
export harvester=~/apes/seeder/build/sdr_harvesting

echo --------------------------------------------
echo Running Flow over Sphere 3D
echo Mesh level: $LEVEL
echo Have fun!
echo --------------------------------------------
echo

export musubi_log=runMusubi.log
export seeder_log=runSeeder.log

# 1 = yes, 0 = no
RUNSEEDER='0'
RUNMUSUBI='1'

if [ ${RUNSEEDER} == "1" ]; then
  rm -rf mesh_L${LEVEL}
  mkdir mesh_L${LEVEL}
  mpiexec -n 1 $seeder | tee ${seeder_log}
  cat mesh_L${LEVEL}/header.lua
  ${harvester}
  echo --------------------------------------------
fi


## Run Musubi
mkdir -p balance
if [ ${RUNMUSUBI} == "1" ]; then
  rm -rf output
  mkdir -p output
  rm -rf balance/*
  rm ${musubi_log}
  rm dbg.00*
  export OMP_NUM_THREADS=1
  # mpiexec -n 7 $musubi | tee ${musubi_log}
  # ls="2 3 4 5 6 7 8"
  range="2 3 4 5 6"
  for i in $range; do
    mpiexec -n ${i} $musubi | tee temp.log
    cat temp.log >> ${musubi_log}
  done
fi
