diff --git a/mfr/core/exceptions.py b/mfr/core/exceptions.py index d977762e..bfc9fc95 100644 --- a/mfr/core/exceptions.py +++ b/mfr/core/exceptions.py @@ -146,6 +146,13 @@ def __init__(self, message, *args, metadata_url: str = '', response: str = '', * 'response': self.response }]) +class CorruptedError(RendererError): + + __TYPE = 'corrupted' + + def __init__(self, *args, renderer_class: str = '', **kwargs): + super().__init__("File is corrupted, impossible to render, please check it's integrity", *args, renderer_class, **kwargs) + class TooBigToRenderError(ProviderError): """If the user tries to render a file larger than a server specified maximum, throw a TooBigToRenderError. diff --git a/mfr/extensions/tabular/libs/__init__.py b/mfr/extensions/tabular/libs/__init__.py index 836e9bda..a79ec321 100644 --- a/mfr/extensions/tabular/libs/__init__.py +++ b/mfr/extensions/tabular/libs/__init__.py @@ -31,10 +31,14 @@ def sav_pandas(): from ..libs.panda_tools import sav_pandas return sav_pandas +def xls(): + from ..libs.xlrd_tools import xls + return xls -def xlsx_xlrd(): - from ..libs.xlrd_tools import xlsx_xlrd - return xlsx_xlrd + +def xlsx(): + from ..libs.xlrd_tools import xlsx + return xlsx def mat_h5py_scipy(): from ..libs.h5py_scipy_tools import mat_h5py_scipy diff --git a/mfr/extensions/tabular/libs/xlrd_tools.py b/mfr/extensions/tabular/libs/xlrd_tools.py index 4c333acc..8ea9e089 100644 --- a/mfr/extensions/tabular/libs/xlrd_tools.py +++ b/mfr/extensions/tabular/libs/xlrd_tools.py @@ -11,7 +11,7 @@ ) -def xlsx_xlrd(fp): +def xls(fp): """ • .xls → xlrd • .xlsx → openpyxl (xlrd ≥2.0 dropped xlsx support) @@ -21,13 +21,11 @@ def xlsx_xlrd(fp): ZipFile) can seek inside safely. """ sheets = OrderedDict() + wb = xlrd.open_workbook(file_contents=to_bytes(fp)) + return parse_xls(wb, sheets) - try: - wb = xlrd.open_workbook(file_contents=to_bytes(fp)) - return parse_xls(wb, sheets) - except xlrd.biffh.XLRDError: - pass - +def xlsx(fp): + sheets = OrderedDict() try: wb = load_workbook(BytesIO(to_bytes(fp)), data_only=True, read_only=True) except zipfile.BadZipFile as exc: diff --git a/mfr/extensions/tabular/settings.py b/mfr/extensions/tabular/settings.py index af6b9b0a..5ca57afc 100644 --- a/mfr/extensions/tabular/settings.py +++ b/mfr/extensions/tabular/settings.py @@ -12,9 +12,9 @@ LIBS = config.get_object('LIBS', { '.csv': [libs.csv_stdlib], '.tsv': [libs.csv_stdlib], - '.gsheet': [libs.xlsx_xlrd], - '.xlsx': [libs.xlsx_xlrd], - '.xls': [libs.xlsx_xlrd], + '.gsheet': [libs.xlsx], + '.xlsx': [libs.xlsx], + '.xls': [libs.xls], '.dta': [libs.dta_pandas], '.sav': [libs.sav_stdlib], '.mat': [libs.mat_h5py_scipy], diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py index c61c3e40..efd4dad8 100644 --- a/mfr/extensions/tabular/utilities.py +++ b/mfr/extensions/tabular/utilities.py @@ -8,7 +8,7 @@ from tempfile import NamedTemporaryFile from mfr.extensions.tabular import compat -from mfr.core.exceptions import SubprocessError, TooBigToRenderError +from mfr.core.exceptions import SubprocessError, TooBigToRenderError, CorruptedError from mfr.extensions.tabular.settings import (PSPP_CONVERT_BIN, PSPP_CONVERT_TIMEOUT) @@ -143,21 +143,13 @@ def parse_xls(wb, sheets): def parse_xlsx(wb, sheets): for name in wb.sheetnames: ws = wb[name] - max_row = ws.max_row or 0 - max_col = ws.max_column or 0 - verify_size(max_row, max_col, '.xlsx') - - if max_row == 0 or max_col == 0: - sheets[name] = ([], []) - continue - header_row = next(ws.iter_rows(max_row=1, values_only=True), []) fields = fix_headers(header_row) rows = [ dict(zip(fields, row)) for row in ws.iter_rows(min_row=2, - max_row=max_row, - max_col=max_col, + max_row=MAX_SIZE, + max_col=MAX_SIZE, values_only=True) ] sheets[name] = (header_population(fields), rows) @@ -165,6 +157,8 @@ def parse_xlsx(wb, sheets): def verify_size(rows, cols, ext): + if rows is None or cols is None: + raise CorruptedError if rows > MAX_SIZE or cols > MAX_SIZE: raise TooBigToRenderError('Table is too large to render.', ext, nbr_cols=cols, nbr_rows=rows) diff --git a/tests/extensions/tabular/test_xlsx_tools.py b/tests/extensions/tabular/test_xlsx_tools.py index a5e761e9..9956e338 100644 --- a/tests/extensions/tabular/test_xlsx_tools.py +++ b/tests/extensions/tabular/test_xlsx_tools.py @@ -9,7 +9,7 @@ class TestTabularPandaTools: def test_xlsx_xlrd(self): with open(os.path.join(BASE, 'files', 'test.xlsx'), 'rb') as fp: - sheets = xlrd_tools.xlsx_xlrd(fp) + sheets = xlrd_tools.xlsx(fp) sheet = sheets.popitem()[1] assert sheet[0][0] == {'field': 'one', 'id': 'one', 'name': 'one', 'sortable': True}