X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/90fe976b26ded0814939ff7ce048aafd8982ae65..3dc17bd27c244c9d2d20cacab6607f2d731046c8:/examples/pyarrow_reading.py diff --git a/examples/pyarrow_reading.py b/examples/pyarrow_reading.py index d67376d..59f9fd9 100644 --- a/examples/pyarrow_reading.py +++ b/examples/pyarrow_reading.py @@ -1,8 +1,8 @@ import pyarrow.dataset as ds -import pyarrow as pa + # A pyarrow dataset abstracts reading, writing, or filtering a parquet file. It does not read dataa into memory. #dataset = ds.dataset(pathlib.Path('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/'), format='parquet', partitioning='hive') -dataset = ds.dataset('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/', format='parquet', partitioning='hive') +dataset = ds.dataset('/gscratch/comdata/output/reddit_comments_by_subreddit.parquet/', format='parquet') # let's get all the comments to two subreddits: subreddits_to_pull = ['seattle','seattlewa']