diff --git a/playbooks/files/scripts/import-to-cassandra.sh b/playbooks/files/scripts/import-to-cassandra.sh index c904cbe..44697df 100644 --- a/playbooks/files/scripts/import-to-cassandra.sh +++ b/playbooks/files/scripts/import-to-cassandra.sh @@ -1,12 +1,12 @@ #!/bin/bash DIR="$( dirname "${BASH_SOURCE[0]}" )" NB_CPUS="$(cat /proc/cpuinfo | grep -i "^processor" | wc -l)" NB_JOBS="$(expr $NB_CPUS "*" 4)" cat /mnt/dataset/revision.csv.gz \ | pigz -d \ | pv --line-mode --average-rate --rate --timer --bytes \ - | parallel -j $NB_JOBS --round-robin --pipe \ + | parallel -j $NB_JOBS --round-robin --pipe --line-buffer \ python3 $DIR/cassandra-bench-tools.py cassandra import-revision-dataset diff --git a/playbooks/files/scripts/import-to-pgsql.sh b/playbooks/files/scripts/import-to-pgsql.sh index 08c68a0..5310b23 100644 --- a/playbooks/files/scripts/import-to-pgsql.sh +++ b/playbooks/files/scripts/import-to-pgsql.sh @@ -1,12 +1,12 @@ #!/bin/bash DIR="$( dirname "${BASH_SOURCE[0]}" )" NB_CPUS="$(cat /proc/cpuinfo | grep -i "^processor" | wc -l)" NB_JOBS="$(expr $NB_CPUS "*" 4)" cat /mnt/dataset/revision.csv.gz \ | pigz -d \ | pv --line-mode --average-rate --rate --timer --bytes \ - | parallel -j $NB_JOBS --round-robin --pipe \ + | parallel -j $NB_JOBS --round-robin --pipe --line-buffer \ python3 $DIR/cassandra-bench-tools.py pgsql import-revision-dataset