added counting functionality to regex code

[mediawiki_dump_tools.git] / wikiq

diff --git a/wikiq b/wikiq
index 6a1ada5cccee93edc9ae4c96f374542b1b4ba118..91b86d758bfc96191fc3b2660dc911b86652f8ff 100755 (executable)
--- a/wikiq
+++ b/wikiq
@@ -34,55 +34,6 @@ def calculate_persistence(tokens_added):
     return(sum([(len(x.revisions)-1) for x in tokens_added]),
            len(tokens_added))
 
     return(sum([(len(x.revisions)-1) for x in tokens_added]),
            len(tokens_added))
 

-def matchmake(scanned_content, rev_data, regex, label):
-    p = re.compile(regex)
-
-    temp_dict = {}
-    # if there are named capture groups in the regex
-    if bool(p.groupindex):
-        capture_groups = list(p.groupindex.keys())
-
-        # initialize the {capture_group_name:list} for each capture group
-        for cap_group in capture_groups:
-            temp_dict["{}_{}".format(label, cap_group)] = []
-
-        # if there are matches of some sort in this revision content, fill the lists for each cap_group
-        if p.search(scanned_content) is not None:
-            m = re.finditer(p,scanned_content)
-            matchobjects = list(m)
-
-            for cap_group in capture_groups:
-                temp_list = []
-                for match in matchobjects:
-                    # we only want to add the match for the capture group if the match is not None
-                    if match.group(cap_group) != None:
-                        temp_list.append(match.group(cap_group))
-
-                # if temp_list of matches is empty just make that column None
-                if len(temp_list)==0:
-                    temp_dict["{}_{}".format(label, cap_group)] = None
-                # else we put in the list we made in the for-loop above
-                else:
-                    temp_dict["{}_{}".format(label, cap_group)] = ', '.join(temp_list)
-        
-        # there are no matches at all in this revision content, we default values to None
-        else:
-            for cap_group in capture_groups:
-                temp_dict["{}_{}".format(label, cap_group)] = None
-
-    # there are no capture groups, we just search for all the matches of the regex
-    else:
-        #given that there are matches to be made
-        if p.search(scanned_content) is not None:
-            m = p.findall(scanned_content)
-            temp_dict[label] = ', '.join(m)
-        else:
-            temp_dict[label] = None    
-    # update rev_data with our new columns
-    rev_data.update(temp_dict)
-#    print(rev_data.keys())
-    return rev_data
-

 
 class WikiqIterator():
     def __init__(self, fh, collapse_user=False):
 
 class WikiqIterator():
     def __init__(self, fh, collapse_user=False):


@@ -176,13 +127,80 @@ class WikiqPage():
     def __next__(self):
         return next(self.__revisions)
 
     def __next__(self):
         return next(self.__revisions)
 

+
+class RegexPair(object):
+    def __init__(self, pattern, label):
+        self.pattern = re.compile(pattern)
+        self.label = label
+        self.has_groups = bool(self.pattern.groupindex)
+        if self.has_groups:
+            self.capture_groups = list(self.pattern.groupindex.keys())
+            
+    def _make_key(self, cap_group):
+        return ("{}_{}".format(self.label, cap_group))
+
+    def matchmake(self, content, rev_data, count_only=False):
+        temp_dict = {}
+        # if there are named capture groups in the regex
+        if self.has_groups:
+
+            # if there are matches of some sort in this revision content, fill the lists for each cap_group
+            if content is not None and self.pattern.search(content) is not None:
+                m = self.pattern.finditer(content)
+                matchobjects = list(m)
+
+                for cap_group in self.capture_groups:
+                    key = self._make_key(cap_group)
+                    temp_list = []
+                    for match in matchobjects:
+                        # we only want to add the match for the capture group if the match is not None
+                        if match.group(cap_group) != None:
+                            temp_list.append(match.group(cap_group))
+
+                    # if temp_list of matches is empty just make that column None
+                    if len(temp_list)==0:
+                        temp_dict[key] = None
+                    # else we put in the list we made in the for-loop above
+                    else:
+                        if count_only:
+                            temp_dict[key] = len(temp_list)
+                        else:
+                            temp_dict[key] = ', '.join(temp_list)
+
+            # there are no matches at all in this revision content, we default values to None
+            else:
+                for cap_group in self.capture_groups:
+                    key = self._make_key(cap_group)
+                    if count_only:
+                        temp_dict[key] = 0
+                    else:
+                        temp_dict[key] = None
+
+        # there are no capture groups, we just search for all the matches of the regex
+        else:
+            #given that there are matches to be made
+            if content is not None and self.pattern.search(content) is not None:
+                m = self.pattern.findall(content)
+                if count_only:
+                    temp_dict[self.label] = len(m)
+                else:
+                    temp_dict[self.label] = ', '.join(m)
+            else:
+                if count_only:
+                    temp_dict[self.label] = 0
+                else:
+                    temp_dict[self.label] = None
+        # update rev_data with our new columns
+        rev_data.update(temp_dict)
+        return rev_data
+
+        

 class WikiqParser():
 class WikiqParser():

-    def __init__(self, input_file, output_file, regex_match_revision, regex_match_comment, regex_revision_label, regex_comment_label, collapse_user=False, persist=None, urlencode=False, namespaces = None, revert_radius=15):
+    def __init__(self, input_file, output_file, regex_match_revision, regex_revision_label, regex_revision_output_count, regex_match_comment, regex_comment_label, regex_comment_output_count, collapse_user=False, persist=None, urlencode=False, namespaces = None, revert_radius=15):

         """ 
         Parameters:
            persist : what persistence method to use. Takes a PersistMethod value
         """
         """ 
         Parameters:
            persist : what persistence method to use. Takes a PersistMethod value
         """

-

         self.input_file = input_file
         self.output_file = output_file
         self.collapse_user = collapse_user
         self.input_file = input_file
         self.output_file = output_file
         self.collapse_user = collapse_user


@@ -191,16 +209,43 @@ class WikiqParser():
         self.namespaces = []
         self.urlencode = urlencode
         self.revert_radius = revert_radius
         self.namespaces = []
         self.urlencode = urlencode
         self.revert_radius = revert_radius

-        self.regex_match_revision = regex_match_revision
-        self.regex_revision_label = regex_revision_label
-        self.regex_match_comment = regex_match_comment
-        self.regex_comment_label = regex_comment_label

 
         if namespaces is not None:
             self.namespace_filter = set(namespaces)
         else:
             self.namespace_filter = None
 
 
         if namespaces is not None:
             self.namespace_filter = set(namespaces)
         else:
             self.namespace_filter = None
 

+        self.regex_revision_pairs = self.make_matchmake_pairs(regex_match_revision, regex_revision_label)
+        self.regex_revision_output_count = regex_revision_output_count
+
+        self.regex_comment_pairs = self.make_matchmake_pairs(regex_match_comment, regex_comment_label)
+        self.regex_comment_output_count = regex_comment_output_count
+
+    def make_matchmake_pairs(self, patterns, labels):
+        if (patterns is not None and labels is not None) and \
+           (len(patterns) == len(labels)):
+            return [RegexPair(pattern, label) for pattern, label in zip(patterns, labels)]
+        elif (patterns is None and labels is None):
+            return []
+        else:
+            sys.exit('Each regular expression *must* come with a corresponding label and vice versa.')
+
+    def matchmake(self, rev, rev_data):
+        rev_data = self.matchmake_revision(rev.text, rev_data)
+        rev_data = self.matchmake_comment(rev.comment, rev_data)
+        return rev_data
+
+    def matchmake_revision(self, text, rev_data):
+        return self.matchmake_pairs(text, rev_data, self.regex_revision_pairs, self.regex_revision_output_count)
+
+    def matchmake_comment(self, comment, rev_data):
+        return self.matchmake_pairs(comment, rev_data, self.regex_comment_pairs, self.regex_comment_output_count)
+
+    def matchmake_pairs(self, text, rev_data, pairs, count_only):
+        for pair in pairs:
+            rev_data = pair.matchmake(text, rev_data, count_only)
+        return rev_data
+

     def __get_namespace_from_title(self, title):
         default_ns = None
 
     def __get_namespace_from_title(self, title):
         default_ns = None
 


@@ -243,10 +288,8 @@ class WikiqParser():
                 if namespace not in self.namespace_filter:
                     continue
 
                 if namespace not in self.namespace_filter:
                     continue
 

-#            print(self.revert_radius)

             rev_detector = mwreverts.Detector(radius = self.revert_radius)
 
             rev_detector = mwreverts.Detector(radius = self.revert_radius)
 

-

             if self.persist != PersistMethod.none:
                 window = deque(maxlen=PERSISTENCE_RADIUS)
 
             if self.persist != PersistMethod.none:
                 window = deque(maxlen=PERSISTENCE_RADIUS)
 


@@ -267,50 +310,17 @@ class WikiqParser():
             for rev in page:
                 
                 # initialize rev_data
             for rev in page:
                 
                 # initialize rev_data

-                rev_data = {}
-
-                # if the command line args only gave a label (and no regular expression is given)
-                if (self.regex_revision_label != None and self.regex_match_revision == None) or (self.regex_comment_label != None and self.regex_match_comment == None):
-                    sys.exit('The given regex label(s) has no corresponding regex to search for.')
-                
-                # if there's anything in the list of regex_match_revision
-                if self.regex_match_revision is not None:
-                    if (self.regex_revision_label == None) or (len(self.regex_match_revision) != len(self.regex_revision_label)):
-                        sys.exit('Each regular expression *must* come with a corresponding label and vice versa.')
-                    
-                    # initialize and construct the list of regex-label tuples
-                    pairs = []
-                    for i in range(0,len(self.regex_match_revision)):
-                        pairs.append((self.regex_match_revision[i], self.regex_revision_label[i]))
-
-                    # for each regex/label pair, we now run matchmake to check and output columns
-                    for pair in pairs:
-                        # pair[0] corresponds to the regex, pair[1] to the label
-                        rev_data = matchmake(rev.text, rev_data, pair[0], pair[1])
-                
-                # if there's anything in the list of regex_match_comment
-                if self.regex_match_comment is not None:
-                    if (self.regex_comment_label == None) or (len(self.regex_match_comment) != len(self.regex_comment_label)):
-                        sys.exit('Each regular expression *must* come with a corresponding label and vice versa.')
-                    
-                    # initialize and construct the list of regex-label tuples
-                    pairs = []
-                    for i in range(0,len(self.regex_match_comment)):
-                        pairs.append((self.regex_match_comment[i], self.regex_comment_label[i]))
-
-                    # for each regex/label pair, we now run matchmake to check and output columns
-                    for pair in pairs:
-                        # pair[0] corresponds to the regex, pair[1] to the label
-                        rev_data = matchmake(rev.comment, rev_data, pair[0], pair[1])
-
-                # we fill out the rest of the data structure now
-                rev_data['revid'] = rev.id
-                rev_data['date_time'] = rev.timestamp.strftime('%Y-%m-%d %H:%M:%S')
-                rev_data['articleid'] = page.id
-                rev_data['editor_id'] = "" if rev.deleted.user == True or rev.user.id is None else rev.user.id
-                rev_data['title'] = '"' + page.title + '"'
-                rev_data['namespace'] = namespace
-                rev_data['deleted'] = "TRUE" if rev.deleted.text else "FALSE"
+                rev_data = {
+                    'revid':rev.id,
+                    'date_time' : rev.timestamp.strftime('%Y-%m-%d %H:%M:%S'),
+                    'articleid' : page.id,
+                    'editor_id' : "" if rev.deleted.user == True or rev.user.id is None else rev.user.id,
+                    'title' : '"' + page.title + '"',
+                    'namespace' : namespace,
+                    'deleted' : "TRUE" if rev.deleted.text else "FALSE"
+                }
+
+                rev_data = self.matchmake(rev, rev_data)

 
                 # if revisions are deleted, /many/ things will be missing
                 if rev.deleted.text:
 
                 # if revisions are deleted, /many/ things will be missing
                 if rev.deleted.text:


@@ -370,6 +380,8 @@ class WikiqParser():
                     rev_data['collapsed_revs'] = rev.collapsed_revs
 
                 if self.persist != PersistMethod.none:
                     rev_data['collapsed_revs'] = rev.collapsed_revs
 
                 if self.persist != PersistMethod.none:

+                    # initialize an empty dictionary before assigning things into it. this catches bugs if the first revision is deleted                    
+                    old_rev_data = {}

                     if rev.deleted.text:
                         for k in ["token_revs", "tokens_added", "tokens_removed", "tokens_window"]:
                             old_rev_data[k] = None
                     if rev.deleted.text:
                         for k in ["token_revs", "tokens_added", "tokens_removed", "tokens_window"]:
                             old_rev_data[k] = None


@@ -497,12 +509,18 @@ parser.add_argument('-RP', '--revision-pattern', dest="regex_match_revision", de
 parser.add_argument('-RPl', '--revision-pattern-label', dest="regex_revision_label", default=None, type=str, action='append',
                     help="The label for the outputted column based on matching the regex in revision text.")
 
 parser.add_argument('-RPl', '--revision-pattern-label', dest="regex_revision_label", default=None, type=str, action='append',
                     help="The label for the outputted column based on matching the regex in revision text.")
 

+parser.add_argument('-RPc', '--revision-pattern-count', dest="regex_revision_output_count", action='store_true',
+                    help="If present, this will cause the revision patterns to return counts of the number of matches instead of the text of the matches themselves.  It will affect all revision patterns.")
+

 parser.add_argument('-CP', '--comment-pattern', dest="regex_match_comment", default=None, type=str, action='append',
                     help="The regular expression to search for in comments of revisions.")
 
 parser.add_argument('-CPl', '--comment-pattern-label', dest="regex_comment_label", default=None, type=str, action='append',
                     help="The label for the outputted column based on matching the regex in comments.")
 
 parser.add_argument('-CP', '--comment-pattern', dest="regex_match_comment", default=None, type=str, action='append',
                     help="The regular expression to search for in comments of revisions.")
 
 parser.add_argument('-CPl', '--comment-pattern-label', dest="regex_comment_label", default=None, type=str, action='append',
                     help="The label for the outputted column based on matching the regex in comments.")
 

+parser.add_argument('-CPc', '--comment-pattern-count', dest="regex_comment_output_count", action='store_true',
+                    help="If present, this will cause the comments patterns to return counts of the number of matches instead of the text of the matches themselves. It will affect all comment patterns.")
+

 args = parser.parse_args()
 
 # set persistence method
 args = parser.parse_args()
 
 # set persistence method


@@ -548,8 +566,10 @@ if len(args.dumpfiles) > 0:
                             revert_radius=args.revert_radius,
                             regex_match_revision = args.regex_match_revision,
                             regex_revision_label = args.regex_revision_label,
                             revert_radius=args.revert_radius,
                             regex_match_revision = args.regex_match_revision,
                             regex_revision_label = args.regex_revision_label,

+                            regex_revision_output_count = args.regex_revision_output_count,

                             regex_match_comment = args.regex_match_comment,
                             regex_match_comment = args.regex_match_comment,

-                            regex_comment_label = args.regex_comment_label)
+                            regex_comment_label = args.regex_comment_label,
+                            regex_comment_output_count = args.regex_comment_output_count)

 
         wikiq.process()
 
 
         wikiq.process()
 


@@ -567,8 +587,11 @@ else:
                         revert_radius=args.revert_radius,
                         regex_match_revision = args.regex_match_revision,
                         regex_revision_label = args.regex_revision_label,
                         revert_radius=args.revert_radius,
                         regex_match_revision = args.regex_match_revision,
                         regex_revision_label = args.regex_revision_label,

+                        regex_revision_output_count = args.regex_revision_output_count,

                         regex_match_comment = args.regex_match_comment,
                         regex_match_comment = args.regex_match_comment,

-                        regex_comment_label = args.regex_comment_label)
+                        regex_comment_label = args.regex_comment_label,
+                        regex_comment_output_count = args.regex_comment_output_count)
+

 
     wikiq.process() 
 
 
     wikiq.process()