From 98c1317af5da5aafd1e7acb31911ca4333312571 Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Fri, 10 Dec 2021 21:23:32 -0800 Subject: [PATCH] update pushshift dumps. --- dumps/check_comments_shas.py | 2 +- dumps/pull_pushshift_comments.sh | 8 ++++---- dumps/pull_pushshift_submissions.sh | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dumps/check_comments_shas.py b/dumps/check_comments_shas.py index 199261c..dd428be 100755 --- a/dumps/check_comments_shas.py +++ b/dumps/check_comments_shas.py @@ -6,7 +6,7 @@ from os import path import hashlib shasums1 = requests.get("https://files.pushshift.io/reddit/comments/sha256sum.txt").text -shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text +#shasums2 = requests.get("https://files.pushshift.io/reddit/comments/daily/sha256sum.txt").text shasums = shasums1 + shasums2 dumpdir = "/gscratch/comdata/raw_data/reddit_dumps/comments" diff --git a/dumps/pull_pushshift_comments.sh b/dumps/pull_pushshift_comments.sh index 40d82d8..ab309dd 100755 --- a/dumps/pull_pushshift_comments.sh +++ b/dumps/pull_pushshift_comments.sh @@ -1,12 +1,12 @@ #!/bin/bash -user_agent='nathante teblunthuis ' +user_agent='"nathante teblunthuis "' output_dir='/gscratch/comdata/raw_data/reddit_dumps/comments' base_url='https://files.pushshift.io/reddit/comments/' -wget -r --no-parent -A 'RC_201*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url -wget -r --no-parent -A 'RC_201*.xz' -U $user_agent -P $output_dir -nd -nc $base_url -wget -r --no-parent -A 'RC_201*.zst' -U $user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RC_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RC_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RC_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url ./check_comments_shas.py diff --git a/dumps/pull_pushshift_submissions.sh b/dumps/pull_pushshift_submissions.sh index 99d89be..36e6473 100755 --- a/dumps/pull_pushshift_submissions.sh +++ b/dumps/pull_pushshift_submissions.sh @@ -1,14 +1,14 @@ #!/bin/bash -user_agent='nathante teblunthuis ' +user_agent='"nathante teblunthuis "' output_dir='/gscratch/comdata/raw_data/reddit_dumps/submissions' base_url='https://files.pushshift.io/reddit/submissions/' -wget -r --no-parent -A 'RS_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url -wget -r --no-parent -A 'RS_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url -wget -r --no-parent -A 'RS_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url -wget -r --no-parent -A 'RS_20*.bz2' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ -wget -r --no-parent -A 'RS_20*.xz' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ -wget -r --no-parent -A 'RS_20*.zst' -U $user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ +wget -r --no-parent -A 'RS_20*.bz2' --user-agent=$user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RS_20*.xz' --user-agent=$user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RS_20*.zst' --user-agent=$user_agent -P $output_dir -nd -nc $base_url +wget -r --no-parent -A 'RS_20*.bz2' --user-agent=$user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ +wget -r --no-parent -A 'RS_20*.xz' --user-agent=$user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ +wget -r --no-parent -A 'RS_20*.zst' --user-agent=$user_agent -P $output_dir -nd -nc $base_url/old_v1_data/ ./check_submission_shas.py -- 2.39.5