civil_comments/get_perspective_scores.py

   1 from googleapiclient import discovery
   2 import json
   3 import csv
   4 from pathlib import Path
   5
   6 from time import sleep
   7
   8 from itertools import islice
   9
  10 API_KEY = open('perspective_api_key').read()
  11
  12 client = discovery.build("commentanalyzer","v1alpha",developerKey=API_KEY,discoveryServiceUrl="https://commentanalyzer.googleapis.com/$discovery/rest?version=v1alpha1",static_discovery=False,)
  13
  14 csvreader = csv.DictReader(open("all_data.csv"), dialect='unix')
  15
  16 outfile = Path("perspective_results.json")
  17 already_scored = set()
  18 if outfile.exists():
  19     already_scored = set([json.loads(l)['id'] for l in open(str(outfile),'r')])
  20
  21 print(f"loaded {len(already_scored)} scored comments")
  22 with open("perspective_results.json",'a') as of:
  23     for line in csvreader:
  24         if line['id'] not in already_scored and len(line.get('comment_text','')) > 0:
  25             analyze_request = {'comment':{'text':line['comment_text']},
  26                                'languages':['en'],
  27                                'requestedAttributes':{'TOXICITY':{},
  28                                                       "SEVERE_TOXICITY":{},
  29                                                       "IDENTITY_ATTACK":{},
  30                                                       "INSULT":{},
  31                                                       "PROFANITY":{},
  32                                                       "THREAT":{}}}
  33             response = client.comments().analyze(body=analyze_request).execute()
  34             response['id'] = line['id']
  35             result = json.dumps(response)
  36             of.write(result + '\n')
  37             of.flush()
  38
  39             sleep(0.10)