Initial commit

[rises_declines_wikia_code.git] / mediawiki_dump_tools / Mediawiki-Utilities / mw / lib / persistence / tests / test_tokenization.py

from nose.tools import eq_

from .. import tokenization


def test_wikitext_split():
    eq_(
        list(tokenization.wikitext_split("foo bar herp {{derp}}")),
        ["foo", " ", "bar", " ", "herp", " ", "{{", "derp", "}}"]
    )