add tests for wikipedia, malformed xml, bzip2, correct bz2 bug in wikiq.

[mediawiki_dump_tools.git] / wikiq

diff --git a/wikiq b/wikiq
index f25874e41b24edf898c4e773d9d8c73581eeddb8..e693cd2ba2dd3ab19ed61505a529f8d216219e8b 100755 (executable)
--- a/wikiq
+++ b/wikiq
@@ -272,12 +272,12 @@ class WikiqParser():
 
 
 def open_input_file(input_filename):
 
 
 def open_input_file(input_filename):

-    if re.match(r'.*\.7z', input_filename):
-        cmd = ["7za", "x", "-so", input_filename, '*.xml'] 
-    elif re.match(r'.*\.gz', input_filename):
-        cmd = ["zcat", input_filename] 
-    elif re.match(r'.*\.bz2', input_filename):
+    if re.match(r'.*\.7z$', input_filename):
+        cmd = ["7za", "x", "-so", input_filename, '*'] 
+    elif re.match(r'.*\.gz$', input_filename):

         cmd = ["zcat", input_filename] 
         cmd = ["zcat", input_filename] 

+    elif re.match(r'.*\.bz2$', input_filename):
+        cmd = ["bzcat", "-dk", input_filename] 

 
     try:
         input_file = Popen(cmd, stdout=PIPE).stdout
 
     try:
         input_file = Popen(cmd, stdout=PIPE).stdout


@@ -322,24 +322,25 @@ if len(args.dumpfiles) > 0:
     for filename in args.dumpfiles:
         input_file = open_input_file(filename)
 
     for filename in args.dumpfiles:
         input_file = open_input_file(filename)
 

-        # open file for output
+        # open directory for output
+        if args.output_dir:
+            output_dir = args.output_dir[0]
+        else:
+            output_dir = "."
+
+        print("Processing file: %s" % filename, file=sys.stderr)
+

         if args.stdout:
             output_file = sys.stdout
         else:
         if args.stdout:
             output_file = sys.stdout
         else:

-            if args.output_dir:
-                output_dir = args.output_dir[0]
-            else:
-                output_dir = "."
-

             filename = os.path.join(output_dir, os.path.basename(filename))
             output_file = open_output_file(filename)
 
         wikiq = WikiqParser(input_file, output_file, 
             filename = os.path.join(output_dir, os.path.basename(filename))
             output_file = open_output_file(filename)
 
         wikiq = WikiqParser(input_file, output_file, 

-                           collapse_user=args.collapse_user,
+                            collapse_user=args.collapse_user,

                             persist=args.persist,
                             urlencode=args.urlencode)
 
                             persist=args.persist,
                             urlencode=args.urlencode)
 

-        print("Processing file: %s" % filename, file=sys.stderr)

 
         wikiq.process()
 
 
         wikiq.process()
 


@@ -348,7 +349,7 @@ if len(args.dumpfiles) > 0:
         output_file.close()
 else:
     wikiq = WikiqParser(sys.stdin, sys.stdout,
         output_file.close()
 else:
     wikiq = WikiqParser(sys.stdin, sys.stdout,

-                       collapse_user=args.collapse_user,
+                        collapse_user=args.collapse_user,

                         persist=args.persist,
                         urlencode=args.urlencode)
     wikiq.process()
                         persist=args.persist,
                         urlencode=args.urlencode)
     wikiq.process()