From 90fe976b26ded0814939ff7ce048aafd8982ae65 Mon Sep 17 00:00:00 2001 From: Nate E TeBlunthuis Date: Tue, 7 Jul 2020 00:51:40 -0700 Subject: [PATCH] Script to demonstrate reading parquet. --- examples/pyarrow_reading.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 examples/pyarrow_reading.py diff --git a/examples/pyarrow_reading.py b/examples/pyarrow_reading.py new file mode 100644 index 0000000..d67376d --- /dev/null +++ b/examples/pyarrow_reading.py @@ -0,0 +1,17 @@ +import pyarrow.dataset as ds +import pyarrow as pa +# A pyarrow dataset abstracts reading, writing, or filtering a parquet file. It does not read dataa into memory. +#dataset = ds.dataset(pathlib.Path('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/'), format='parquet', partitioning='hive') +dataset = ds.dataset('/gscratch/comdata/output/reddit_submissions_by_subreddit.parquet/', format='parquet', partitioning='hive') + +# let's get all the comments to two subreddits: +subreddits_to_pull = ['seattle','seattlewa'] + +# a table is a low-level structured data format. This line pulls data into memory. Setting metadata_n_threads > 1 gives a little speed boost. +table = dataset.to_table(filter = ds.field('subreddit').isin(subreddits_to_pull), columns=['id','subreddit','CreatedAt','author','ups','downs','score','subreddit_id','stickied','title','url','is_self','selftext']) + +# Since data from just these 2 subreddits fits in memory we can just turn our table into a pandas dataframe. +df = table.to_pandas() + +# We should save this smaller dataset so we don't have to wait 15 min to pull from parquet next time. +df.to_csv("mydataset.csv") -- 2.39.5