X-Git-Url: https://code.communitydata.science/cdsc_reddit.git/blobdiff_plain/f0176d9f0de93f0e4f3ab1d676c852c2e5fad3b3..87ffaa6858919bd830694d60dd4fc7b1857b462a:/similarities/tfidf.py?ds=inline diff --git a/similarities/tfidf.py b/similarities/tfidf.py index 30033a8..002e89f 100644 --- a/similarities/tfidf.py +++ b/similarities/tfidf.py @@ -15,10 +15,9 @@ def _tfidf_wrapper(func, inpath, outpath, topN, term_colname, exclude, included_ else: include_subs = select_topN_subreddits(topN) - df = func(df, include_subs, term_colname) - - df.write.parquet(outpath,mode='overwrite',compression='snappy') + dfwriter = func(df, include_subs, term_colname) + dfwriter.parquet(outpath,mode='overwrite',compression='snappy') spark.stop() def tfidf(inpath, outpath, topN, term_colname, exclude, included_subreddits):