#!/bin/sh

echo "Starting script for importing dumps"
echo "Reading dumps to process from dumps_to_import.txt"
while read -r line ; do
    # Initialize folder, unpack and sort. $line has the full path
		echo "Processing file: $line"
		folder_new_name=$(basename $line)
		folder_new_name="${folder_new_name%%.*}"
    echo "Name of folder: $folder_new_name"
		
		# Create dir (if it not exists already)
		mkdir $folder_new_name
		echo "Importing file..."
		TEMPDIR=$folder_new_name
		LOGDIR=$folder_new_name
		UNSORTED_KGTK=unsorted.tsv.gz
		KGTK_FLAGS="--debug --timing --progress --progress-tty `tty`"
		VERBOSE="--verbose"
		# ==============================================================================
		# Import the Wikidata dump file, getting labels, aliases, and descriptions
		# in English and in all languages.
		kgtk  --debug --timing --progress import-wikidata \
        -i $line \
        --node ${TEMPDIR}/nodefile.tsv \
        --edge ${TEMPDIR}/edgefile.tsv \
        --qual ${TEMPDIR}/qualfile.tsv \
        --use-mgzip-for-input True \
        --use-mgzip-for-output True \
        --use-shm True \
        --procs 6 \
        --mapper-batch-size 5 \
        --max-size-per-mapper-queue 3 \
        --deprecated \
        --single-mapper-queue True \
        --collect-results True \
        --collect-seperately True\
        --collector-batch-size 10 \
        --collector-queue-per-proc-size 3 \
        --progress-interval 500000 --fail-if-missing False
		#		| tee ${LOGDIR}/import_wikidata.log
		#kgtk import-wikidata \
    #    -i $line \
    #    --node "$folder_new_name"/nodefile.tsv \
    #    --edge "$folder_new_name"/edgefile.tsv \
    #    --qual "$folder_new_name"/qualfile.tsv \
    #    --use-mgzip-for-input True \
    #    --use-mgzip-for-output True \
    #    --use-shm True \
    #    --procs 12 \
    #    --mapper-batch-size 5 \
    #    --max-size-per-mapper-queue 3 \
    #    --single-mapper-queue True \
    #    --collect-results True \
    #    --collect-seperately True\
    #    --collector-batch-size 10 \
    ## is --progress-tty `tty`    --collector-queue-per-proc-size 3 \
    #    --progress-interval 500000
		
		# Note: Sorting is a separate process because in some cases it has led to removing exported files 
		
		#echo "Sorting file ..."
		#kgtk sort -i "$folder_new_name"/edgefile.tsv -c 'id' -o "$folder_new_name"/edgefile_sorted.tsv
		# Remove edge file (to save a little space)
		#rm "$folder_new_name"/edgefile.tsv
done < dumps_to_import.txt
#dumps_to_import.txt
echo "COMPLETED. Now execute sort_dumps.sh to sort the kgtk files and then calculate deltas"
