preprocess

#!/bin/bash

#===============================================================================
#       USAGE:  ./preprocess <judgeclass> <verbose> <topic> <query>
#
# DESCRIPTION: Creates and loads start up files, creates seed trainsets.
#===============================================================================

source "${UTIL_PATH}/handle_errors"
source "${UTIL_PATH}/colors"

JUDGECLASS=$1; shift
VERBOSE=false; [[ $1 == "true" ]] && VERBOSE=true; shift
TOPIC=$1; shift
QUERY=$1; shift

#----------------------------------------------------------------------
# if data is already prepared, skips data extraction
#----------------------------------------------------------------------
if [ ! -f "${JUDGECLASS}.svm.fil" ]; then
    pushd Corpus

    e_primary "Preparing dataset (executing dofast)..."
    ./dofast "$JUDGECLASS" $VERBOSE
    e_success "Done."

    cp "$JUDGECLASS".df ../"$JUDGECLASS".df > /dev/null

    cp "$JUDGECLASS".svm.fil ../"$JUDGECLASS".svm.fil

    e_primary "Running feature compression SVD..."
    ../svd/./script_create_svd.sh  ../$JUDGECLASS.svm.fil 10
    e_success "Done."

    popd
fi

RESULT_DIR="result/${JUDGECLASS}"

$VERBOSE && e_primary "Creating directories to store results..."

try
(
    rm -rf "${RESULT_DIR}/${TOPIC}/"
    mkdir -p "${RESULT_DIR}/${TOPIC}/"

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

$VERBOSE && e_primary "Creating topic directory..."

try
(
    rm -rf $TOPIC
    mkdir $TOPIC

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

$VERBOSE && e_primary "Creating goldendb..."

try
(
cat judgement/qrels.$JUDGECLASS.list \
    | grep "$TOPIC" \
    | cut -d' ' -f3 \
    | sort > $TOPIC/goldendb

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

$VERBOSE && e_primary "Creating file with total number of docs..."

try
(
    echo `wc -l < "$JUDGECLASS".svm.fil` > N

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

pushd $TOPIC

echo "$QUERY" > "$TOPIC".seed.doc

ln -n ../$JUDGECLASS.svm.fil $JUDGECLASS.svm.fil

$VERBOSE && echo "Preparing 'docfils' file with all documents..."

try
(
    cut -d' ' -f1 ../$JUDGECLASS.svm.fil \
        | sed -e 's/.*/& &/' > docfil

    cut -d' ' -f1 docfil \
        | cat -n > docfils

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

$VERBOSE && echo "Preparing auxiliary and result files..."

try
(
    touch rel.$TOPIC.fil
    touch prel.$TOPIC

    rm -rf prevalence.rate
    touch prevalence.rate

    rm -rf rel.rate
    touch rel.rate

    rm -f new[0-9][0-9].$TOPIC tail[0-9][0-9].$TOPIC self*.$TOPIC gold*.$TOPIC
    touch new00.$TOPIC

    touch seed

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

cp $TOPIC.seed.doc ../$TOPIC.seed.doc

popd

$VERBOSE && e_primary "Executing ./dofeaturesseed..."
./dofeaturesseed $TOPIC.seed.doc $TOPIC $JUDGECLASS $VERBOSE
$VERBOSE && e_success "Finished ./dofeaturesseed."

pushd $TOPIC

# TODO: Adds error detection here.
$VERBOSE && e_primary "Preparing ../$JUDGECLASS.svm.fil (runs ../dosplit)..."

sed -e 's/[^ ]*/0/' ../$JUDGECLASS.svm.fil \
    | ../dosplit

$VERBOSE && e_success "Done."

$VERBOSE && e_primary "Preparing $TOPIC.synthetic.seed..."

try
(
    sed -e 's/[^ ]*/1/' svm.$TOPIC.seed.doc.fil > $TOPIC.synthetic.seed

) 2> $STD_ERROR_OUT

catch || {
    exit_on_error
}

$VERBOSE && e_success "Done."

#-------------------------------------------------------------------------
# set kissdb parameters and execute it if database doesn't exists
#-------------------------------------------------------------------------
KEYSIZE=$(awk 'BEGIN{a=0}{len = length($1); a=a<len?len:a}END{print a}' \
    $JUDGECLASS.svm.fil)
VALSIZE=$(awk 'BEGIN{a=0}{len = length($0); a=a<len?len:a}END{print a}' \
    $JUDGECLASS.svm.fil)
export KEYSIZE=$((KEYSIZE+2))
export VALSIZE=$((VALSIZE+2))

if [ ! -f "../$JUDGECLASS.db" ]; then
    $VERBOSE && e_primary "Indexing $JUDGECLASS, keysize = $KEYSIZE, valsize = $VALSIZE"

    try
    (
        ../indexer $JUDGECLASS.svm.fil "$JUDGECLASS".db $KEYSIZE $VALSIZE

    ) 2> $STD_ERROR_OUT

    catch || {
        exit_on_error
    }

    $VERBOSE && e_success "Done."
fi

popd