From: Nate E TeBlunthuis Date: Wed, 28 Jul 2021 22:32:04 +0000 (-0700) Subject: no longer do we need to get daily dumps X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/commitdiff_plain/596e1ff339db30045e0811588557db0492e3af26 no longer do we need to get daily dumps --- diff --git a/dumps/pull_pushshift_comments.sh b/dumps/pull_pushshift_comments.sh index 3f6d2c9..40d82d8 100755 --- a/dumps/pull_pushshift_comments.sh +++ b/dumps/pull_pushshift_comments.sh @@ -8,7 +8,5 @@ wget -r --no-parent -A 'RC_201*.bz2' -U $user_agent -P $output_dir -nd -nc $base wget -r --no-parent -A 'RC_201*.xz' -U $user_agent -P $output_dir -nd -nc $base_url wget -r --no-parent -A 'RC_201*.zst' -U $user_agent -P $output_dir -nd -nc $base_url -# starting in 2020 we use daily dumps not monthly dumps -wget -r --no-parent -A 'RC_202*.gz' -U $user_agent -P $output_dir -nd -nc $base_url/daily/ ./check_comments_shas.py