#!/bin/bash

set -e

TOPDIR=`pwd`

# pegasus bin directory is needed to find keg
BIN_DIR=`pegasus-config --bin`
CONDOR_POOL_PEGASUS_HOME=`dirname $BIN_DIR`

# build the dax generator
PYTHONPATH=`pegasus-config --python`
export PYTHONPATH=".:$PYTHONPATH"


# generate the dax
./local_hierarchy.py $CONDOR_POOL_PEGASUS_HOME  > outer.dax

mkdir -p input

INNER_DAX_PROPERTIES=${TOPDIR}/input/inner.properties
INNER_DAX_TC=${TOPDIR}/input/inner.tc.text
INNER_DAX_RC=${TOPDIR}/input/inner.rc.data
INNER_DAX_SC=${TOPDIR}/input/inner.sites.xml

# create the replica catalog for outer level workflow
# specifying the locations of various configuration files
# pegasus requires for the inner level workflow
echo "dax1.properties  $INNER_DAX_PROPERTIES site=local" > rc.data
echo "dax1.rc  $INNER_DAX_RC site=local" >> rc.data
echo "dax1.tc.text $INNER_DAX_TC site=local" >> rc.data
echo "dax1.sites.xml $INNER_DAX_SC site=local" >> rc.data
echo "inner2.dax  $TOPDIR/inner2.dax site=local" >> rc.data

#create the replica catalog for the inner dax
cat > f.a <<EOF
Sample input file for the first inner dax job.

EOF
echo "f.a $TOPDIR/f.a site="local"" > $INNER_DAX_RC

#generate the properties file for the inner workflow
cat > $INNER_DAX_PROPERTIES <<EOF
pegasus.catalog.site.file=dax1.sites.xml

pegasus.catalog.transformation=Text
pegasus.catalog.transformation.file=dax1.tc.text

pegasus.catalog.replica=File
pegasus.catalog.replica.file=dax1.rc


pegasus.dir.storage.deep=false

pegasus.condor.logs.symlink = false

EOF


#create the transformation catalog for the outer level workflow
cat >tc.text <<EOF
tr level1::sleep { 
  site local {
    pfn "/bin/sleep"
    os "linux"
    type "INSTALLED"
  }
}

tr level2::sleep {   
  site local {
    pfn "/bin/sleep"
    os "linux"
    type "INSTALLED"
  }
}

# assumption is that the submit directory is accessbile on the nodes
# making up the condor pool. path can be updated to a local path
# on the nodes
tr blackdiamond::generate{
   site condorpool {
    pfn "$TOPDIR/blackdiamond.py"     
    os "linux"
    type "INSTALLED"
   }
}
EOF

#create the transformation catalog for inner dax
# In this case the binary is keg, which is shipped with Pegasus, so we use
# Assumption is that pegasus-keg is installed at same place on the submit host
# and the nodes in the condorpool.
cat > $INNER_DAX_TC <<EOF

tr level2::sleep {   
  site local {
    pfn "/bin/sleep"
    os "linux"
    type "INSTALLED"
  }
}

tr diamond::preprocess:4.0{
   site condorpool {
    pfn "$BIN_DIR/pegasus-keg"
    os "linux"
    type "INSTALLED"
   }
}

tr diamond::analyze:4.0{
   site condorpool {
    pfn "$BIN_DIR/pegasus-keg"
    os "linux"
    type "INSTALLED"
   }
}
tr diamond::findrange:4.0{
   site condorpool {
    pfn "$BIN_DIR/pegasus-keg"
    os "linux"
    type "INSTALLED"
   }
}
EOF

# create the site catalog for both outer level and inner level workflow
cat >sites.xml <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<sitecatalog xmlns="http://pegasus.isi.edu/schema/sitecatalog" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pegasus.isi.edu/schema/sitecatalog http://pegasus.isi.edu/schema/sc-4.0.xsd" version="4.0">

    <site  handle="local" arch="x86" os="LINUX">
        <directory type="shared-scratch" path="$TOPDIR/work/local-site/scratch">
            <file-server operation="all" url="file://$TOPDIR/work/local-site/scratch"/>
        </directory>
        <directory type="local-storage" path="$TOPDIR/outputs/local-site">
            <file-server operation="all" url="file://$TOPDIR/outputs/local-site"/>
        </directory>
        <!-- <profile namespace="env" key="CONDOR_HOME">/opt/condor/7.8.8</profile>-->
    </site>

    <site  handle="condorpool" arch="x86" os="LINUX">
        <directory type="shared-scratch" path="$TOPDIR/work/condorpool/scratch">
            <file-server operation="all" url="file://$TOPDIR/work/condorpool/scratch"/>
        </directory>
        <directory type="local-storage" path="$TOPDIR/outputs/condorpool">
            <file-server operation="all" url="file://$TOPDIR/outputs/condorpool"/>
        </directory>
        <profile namespace="pegasus" key="style" >condor</profile>
        <profile namespace="condor" key="universe" >vanilla</profile>
        <profile namespace="env" key="PEGASUS_HOME">$CONDOR_POOL_PEGASUS_HOME</profile>
    </site>

</sitecatalog>
EOF

#use the same site catalog file for the inner level workflow
cp sites.xml $INNER_DAX_SC


# plan and submit the  workflow
pegasus-plan \
    --conf pegasusrc \
    --sites local,condorpool \
    --dir work \
    --output-site local \
    --dax outer.dax \
#    --submit 

