From 596e1ff339db30045e0811588557db0492e3af26 Mon Sep 17 00:00:00 2001 From: Nate E TeBlunthuis Date: Wed, 28 Jul 2021 15:32:04 -0700 Subject: [PATCH] no longer do we need to get daily dumps --- dumps/pull_pushshift_comments.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/dumps/pull_pushshift_comments.sh b/dumps/pull_pushshift_comments.sh index 3f6d2c9..40d82d8 100755 --- a/dumps/pull_pushshift_comments.sh +++ b/dumps/pull_pushshift_comments.sh @@ -8,7 +8,5 @@ wget -r --no-parent -A 'RC_201*.bz2' -U $user_agent -P $output_dir -nd -nc $base wget -r --no-parent -A 'RC_201*.xz' -U $user_agent -P $output_dir -nd -nc $base_url wget -r --no-parent -A 'RC_201*.zst' -U $user_agent -P $output_dir -nd -nc $base_url -# starting in 2020 we use daily dumps not monthly dumps -wget -r --no-parent -A 'RC_202*.gz' -U $user_agent -P $output_dir -nd -nc $base_url/daily/ ./check_comments_shas.py -- 2.39.5