# /bin/bash

# This script creates a dataset of those statements that have been deleted and not added afterwards.
# We do this by adding the deleted statements and calculating the difference between the total aggregated
# and each of the added files for each delta. This way if something has been added after being deleted, it
# will be filtered out.

folder=deltas
line_n=0
removed=removed_statements.tsv
removed_aux=removed_statements_aux.tsv
for entry in "$folder"/*
do
  if [ $line_n -lt 1 ]
  then
		echo "Initializing. Concatenating $entry/deleted.tsv as $removed" 			
		kgtk cat -i $entry/deleted.tsv -o $removed	
  fi
  if [ $line_n -gt 0 ]
  then
			echo "Calculating difference with entry $entry"
      # Calculate the difference with added statements
			kgtk ifnotexists -i $removed --filter-on "$entry"/added.tsv -o $removed_aux
			# Aggregate the difference into the deleted file
			echo "Concatenating statements at the end of $removed"
			kgtk cat -i $removed_aux "$entry"/deleted.tsv  -o $removed
			# Precaution just in case there are problems reading/writing in the same files
			rm $removed_aux
	fi
  line_n=`expr $line_n + 1`
done
