Browse Source

Parser for Perl's Data::Dumper output with tests

Added testsuite and coverage report
master
Johann Schmitz 6 years ago
parent
commit
642852e20f
Signed by: ercpe GPG Key ID: A084064277C501ED
  1. 2
      .gitignore
  2. 9
      Makefile
  3. 6
      requirements.txt
  4. 108
      src/gitbrowser/utils/perlcrap.py
  5. 2
      tests/__init__.py
  6. 32
      tests/settings.py
  7. 119
      tests/test_perlcrap.py

2
.gitignore

@ -3,3 +3,5 @@
__pycache__
src/gitbrowser/settings/dev.py
.coverage

9
Makefile

@ -0,0 +1,9 @@
TARGET?=tests
test:
DJANGO_SETTINGS_MODULE=tests.settings PYTHONPATH=".:src" django-admin.py test ${TARGET} -v2
coverage:
coverage erase
PYTHONPATH="." coverage run --source='src' src/manage.py test tests --settings "tests.settings"
coverage report

6
requirements.txt

@ -3,4 +3,8 @@ git-python
natsort
bootstrap3
pygments
markdown
markdown
# test coverage
coverage

108
src/gitbrowser/utils/perlcrap.py

@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
import re
import shlex
class DataDumperReader(object):
def read(self, filename):
with open(filename, 'r') as f:
file_content = f.read()
return self.parse(file_content)
def parse(self, raw):
data = raw
py_data = {}
while ';' in data:
m = re.match('(.*?);$', data, re.MULTILINE | re.DOTALL)
current_data = m.group(1).strip()
py_data.update([self.parse_block(current_data)])
data = data[m.end(1) + 1:]
assert data.strip() == "", 'Leftovers found: %s' % data
return py_data
def parse_block(self, block_data):
lexer = shlex.shlex(block_data)
lexer.quotes = "'"
lexer.wordchars += '"'
return self.parse_structure(lexer)
def unquote(self, s):
if len(s or "") <= 2:
return s
if s[0] == s[-1] and s[0] == "'":
return s[1:-1]
return s
def parse_structure(self, lexer):
token = lexer.next()
if token == '%':
variable_name = lexer.next()
nt = lexer.next()
assert nt == "=", "Expected equal sign; got %s" % nt
return self.unquote(variable_name), self.parse_dict(lexer)
if token == '(' or token == '{':
lexer.push_token(token)
return self.parse_dict(lexer)
if token == '[':
lexer.push_token(token)
return self.parse_list(lexer)
if token == '$':
variable_name = lexer.next()
nt = lexer.next()
assert nt == "=", "Expected equal sign; got %s" % nt
return variable_name, lexer.next()
return self.unquote(token)
def parse_dict(self, lexer):
token = lexer.next()
assert token in ('(', '{'), "Expected '(' or '{'; got %s" % token
d = {}
for token in lexer:
if token == ')' or token == '}':
break
if token == ',':
continue
variable_name = token
nt = lexer.next() + lexer.next()
assert nt == "=>", "Expected '=>'; got %s" % nt
d[self.unquote(variable_name)] = self.parse_structure(lexer)
return d
def parse_list(self, lexer):
token = lexer.next()
assert token == '[', "Expected '['; got %s" % token
l = []
for token in lexer:
if token == ']':
return l
if token == ',':
continue
lexer.push_token(token)
list_item = self.parse_structure(lexer)
l.append(list_item)
nt = lexer.next()
if nt == ']':
break
assert nt == ",", "Expected ','; got %s" % nt
return l

2
tests/__init__.py

@ -0,0 +1,2 @@
# -*- coding: utf-8 -*-

32
tests/settings.py

@ -0,0 +1,32 @@
# -*- coding: utf-8 -*-
import os
BASE_DIR = os.path.dirname(__file__)
SECRET_KEY = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890'
INSTALLED_APPS = [
'django.contrib.auth',
'django.contrib.admin',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'tests',
]
MIDDLEWARE_CLASSES = (
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
)
ROOT_URLCONF = 'tests.urls'
DATABASES = {}
TEMPLATE_DIRS = (
os.path.join(BASE_DIR, 'templates'),
)

119
tests/test_perlcrap.py

@ -0,0 +1,119 @@
# -*- coding: utf-8 -*-
from django.test.testcases import SimpleTestCase
from gitbrowser.utils.perlcrap import DataDumperReader
class PerlCrapTestCase(SimpleTestCase):
def parse_and_assert(self, s, reader=None):
reader = reader or DataDumperReader()
obj = reader.parse(s)
self.assertTrue(isinstance(obj, dict))
self.assertGreater(len(obj), 0)
return obj
def parse_and_assert_hash(self, s, hash_name='my_hash', assert_isinstance=None, reader=None):
d = self.parse_and_assert(s, reader)
self.assertEqual(len(d), 1)
self.assertEqual(d.keys()[0], hash_name)
obj = d[hash_name]
if assert_isinstance:
self.assertTrue(isinstance(obj, assert_isinstance))
return obj
def test_parse_scalar(self):
s = "$foobar = 1;"
d = self.parse_and_assert(s)
self.assertEqual(len(d), 1)
self.assertEqual(d.keys()[0], 'foobar')
self.assertEqual(d['foobar'], '1')
def test_invalid_input(self):
s = "foobar"
try:
d = self.parse_and_assert(s)
self.fail("%s should not parse successfully" % s)
except AssertionError as ae:
if not str(ae).startswith('Leftovers found:'):
self.fail('Wrong AssertionError found')
except Exception as ex:
self.fail("A %s should not be raised" % type(ex))
def test_parse_hash(self):
s = "%foobar = ();"
d = self.parse_and_assert_hash(s, 'foobar', dict)
self.assertEqual(len(d), 0)
def test_parse_list(self):
s = """%my_hash = (
'foobar' => [ 1, 2, 3 ]
);"""
d = self.parse_and_assert_hash(s, assert_isinstance=dict)
self.assertEqual(len(d), 1)
self.assertEqual(d.keys()[0], 'foobar')
l = d['foobar']
self.assertTrue(isinstance(l, list))
self.assertEqual(len(l), 3)
self.assertEqual(l, ['1', '2', '3'])
def test_nested_lists(self):
s = """%my_hash = (
'foobar' => [ [ 1, 2, 3 ] ]
);"""
d = self.parse_and_assert_hash(s, assert_isinstance=dict)
self.assertEqual(len(d), 1)
self.assertEqual(d.keys()[0], 'foobar')
l = d['foobar']
self.assertTrue(isinstance(l, list))
self.assertEqual(len(l), 1)
self.assertEqual(l[0], ['1', '2', '3'])
def test_unquote(self):
reader = DataDumperReader()
# too short to contain a leading and a trailing quote
self.assertEqual(reader.unquote(None), None)
self.assertEqual(reader.unquote(""), "")
self.assertEqual(reader.unquote("a"), "a")
# no quotes
self.assertEqual(reader.unquote("aaaa"), "aaaa")
# incomplete or wrong quote position
self.assertEqual(reader.unquote("'aa"), "'aa")
self.assertEqual(reader.unquote("a'a"), "a'a")
self.assertEqual(reader.unquote("aa'"), "aa'")
# wrong quotes (double)
self.assertEqual(reader.unquote('"a"'), '"a"')
# quotes at the first and last position
self.assertEqual(reader.unquote("'aa'"), "aa")
def test_multi(self):
s = """$foo = 1;
$bar = 1;
$baz = 1;"""
d = self.parse_and_assert(s)
self.assertEqual(len(d), 3)
self.assertEqual(sorted(d.keys()), ['bar', 'baz', 'foo'])
for x in ['bar', 'baz', 'foo']:
self.assertEqual(d[x], '1')