]> code.communitydata.science - covid19.git/blob - keywords/src/collect_trends.py
add documentation for the output files
[covid19.git] / keywords / src / collect_trends.py
1 # this follows a similar approach to nick's trends.js but in python
2 from pytrends.request import TrendReq
3 from datetime import datetime
4 from os import path
5 import csv
6 from itertools import islice, chain, zip_longest
7 import pandas as pd
8
9
10 # from itertools recipes
11 #https://docs.python.org/3.6/library/itertools.html#itertools-recipes
12 def grouper(iterable, n, fillvalue=None):
13     "Collect data into fixed-length chunks or blocks"
14     # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
15     args = [iter(iterable)] * n
16     return zip_longest(*args, fillvalue=fillvalue)
17
18 def get_daily_trends():
19     trendReq = TrendReq(backoff_factor=0.2)
20     today_trending = trendReq.today_searches()
21     daily_trends_outfile = path.join("..","output","daily_google_trends.csv")
22
23     write_header = False
24     header = ['date','term','top']
25
26     if not path.exists(daily_trends_outfile):
27         write_header = True
28
29     with open("../output/intermediate/daily_google_trends.csv",'a',newline='') as of:
30         writer = csv.writer(of)
31         if write_header:
32             writer.writerow(header)
33
34         for i, trend in enumerate(today_trending):
35             writer.writerow([str(datetime.now().date()),trend,i])
36
37 def get_related_queries(stems):
38     # we have to batch these in sets of 5
39     trendReq = TrendReq(backoff_factor=0.2)
40     def _get_related_queries(chunk):
41         kw_list = list(filter(lambda x: x is not None, chunk))
42         trendReq.build_payload(kw_list=kw_list)
43         related_queries = trendReq.related_queries()
44         for term, results in related_queries.items():
45             for key, df in results.items():
46                 if df is not None:
47                     df["term"] = term
48                 yield (key,df)
49
50     l = chain(*map(_get_related_queries, grouper(stems,5)))
51     out = {}
52     for key, value in l:
53         if key in out:
54             out[key].append(value)
55         else:
56             out[key] = [value]
57
58     for k in out.keys():
59         df = pd.concat(out[k])
60         df['date'] = str(datetime.now().date())
61         out[k] = df
62         outfile = path.join('..','output','intermediate',f"related_searches_{k}.csv")
63         if path.exists(outfile):
64             mode = 'a'
65             header = False
66         else:
67             mode = 'w'
68             header = True
69
70         df.to_csv(outfile, mode=mode, header=header,index=False)
71
72 stems = [t.strip() for t in open("../resources/base_terms.txt",'r')]
73
74 get_daily_trends()
75
76 get_related_queries(stems)

Community Data Science Collective || Want to submit a patch?