client = discovery.build("commentanalyzer","v1alpha",developerKey=API_KEY,discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",static_discovery=False,)
-csvreader = csv.DictReader(open("all_data.csv"))
+csvreader = csv.DictReader(open("all_data.csv"), dialect='unix')
outfile = Path("perspective_results.json")
already_scored = set()
if outfile.exists():
already_scored = set([json.loads(l)['id'] for l in open(str(outfile),'r')])
-with open("perspective_results.json",'w') as of:
+print(f"loaded {len(already_scored)} scored comments")
+with open("perspective_results.json",'a') as of:
for line in csvreader:
- if line['id'] not in already_scored:
+ if line['id'] not in already_scored and len(line.get('comment_text','')) > 0:
analyze_request = {'comment':{'text':line['comment_text']},
'languages':['en'],
'requestedAttributes':{'TOXICITY':{},