import io from multiprocessing import Queue from nose.tools import eq_, raises from ..processor import DONE, Processor SAMPLE_XML = """ Wikipedia http://en.wikipedia.org/wiki/Main_Page MediaWiki 1.22wmf2 first-letter Talk Foo 0 1 1 2004-08-09T09:04:08Z Gen0cide 92182 Revision 1 text g9chqqg94myzq11c56ixvq7o1yg75n9 wikitext text/x-wiki 2 2004-08-10T09:04:08Z 222.152.210.109 Revision 2 text g9chqqg94myzq11c56ixvq7o1yg75n9 wikitext Comment 2 text/x-wiki Bar 1 2 3 2004-08-11T09:04:08Z 222.152.210.22 Revision 3 text g9chqqg94myzq11c56ixvq7o1yg75n9 wikitext text/x-wiki """ def test_processor(): pathq = Queue() pathq.put(io.StringIO(SAMPLE_XML)) outputq = Queue() def process_dump(dump, path): for page in dump: yield page.id processor = Processor(pathq, outputq, process_dump) processor.start() error, item = outputq.get() assert not error eq_(item, 1) error, item = outputq.get() assert not error eq_(item, 2) error, item = outputq.get() assert not error eq_(item, DONE) def test_processor_error(): pathq = Queue() pathq.put(io.StringIO(SAMPLE_XML)) outputq = Queue() def process_dump(dump, path): raise Exception("foo") processor = Processor(pathq, outputq, process_dump) processor.start() error, item = outputq.get() assert error error, item = outputq.get() assert not error eq_(item, DONE)