From 838872b94e4cea1e1830c41b419a231dcf23a68d Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 1 Aug 2025 15:53:05 +0300 Subject: [PATCH 1/3] made xslx files rendering first 10000 lines instead of not rendering them at all --- mfr/core/exceptions.py | 7 +++++++ mfr/extensions/tabular/libs/__init__.py | 10 +++++++--- mfr/extensions/tabular/libs/xlrd_tools.py | 12 +++++------- mfr/extensions/tabular/settings.py | 6 +++--- mfr/extensions/tabular/utilities.py | 18 ++++++------------ 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/mfr/core/exceptions.py b/mfr/core/exceptions.py index d977762ee..bfc9fc953 100644 --- a/mfr/core/exceptions.py +++ b/mfr/core/exceptions.py @@ -146,6 +146,13 @@ def __init__(self, message, *args, metadata_url: str = '', response: str = '', * 'response': self.response }]) +class CorruptedError(RendererError): + + __TYPE = 'corrupted' + + def __init__(self, *args, renderer_class: str = '', **kwargs): + super().__init__("File is corrupted, impossible to render, please check it's integrity", *args, renderer_class, **kwargs) + class TooBigToRenderError(ProviderError): """If the user tries to render a file larger than a server specified maximum, throw a TooBigToRenderError. diff --git a/mfr/extensions/tabular/libs/__init__.py b/mfr/extensions/tabular/libs/__init__.py index 836e9bda0..a79ec321b 100644 --- a/mfr/extensions/tabular/libs/__init__.py +++ b/mfr/extensions/tabular/libs/__init__.py @@ -31,10 +31,14 @@ def sav_pandas(): from ..libs.panda_tools import sav_pandas return sav_pandas +def xls(): + from ..libs.xlrd_tools import xls + return xls -def xlsx_xlrd(): - from ..libs.xlrd_tools import xlsx_xlrd - return xlsx_xlrd + +def xlsx(): + from ..libs.xlrd_tools import xlsx + return xlsx def mat_h5py_scipy(): from ..libs.h5py_scipy_tools import mat_h5py_scipy diff --git a/mfr/extensions/tabular/libs/xlrd_tools.py b/mfr/extensions/tabular/libs/xlrd_tools.py index 4c333accd..8ea9e0899 100644 --- a/mfr/extensions/tabular/libs/xlrd_tools.py +++ b/mfr/extensions/tabular/libs/xlrd_tools.py @@ -11,7 +11,7 @@ ) -def xlsx_xlrd(fp): +def xls(fp): """ • .xls → xlrd • .xlsx → openpyxl (xlrd ≥2.0 dropped xlsx support) @@ -21,13 +21,11 @@ def xlsx_xlrd(fp): ZipFile) can seek inside safely. """ sheets = OrderedDict() + wb = xlrd.open_workbook(file_contents=to_bytes(fp)) + return parse_xls(wb, sheets) - try: - wb = xlrd.open_workbook(file_contents=to_bytes(fp)) - return parse_xls(wb, sheets) - except xlrd.biffh.XLRDError: - pass - +def xlsx(fp): + sheets = OrderedDict() try: wb = load_workbook(BytesIO(to_bytes(fp)), data_only=True, read_only=True) except zipfile.BadZipFile as exc: diff --git a/mfr/extensions/tabular/settings.py b/mfr/extensions/tabular/settings.py index af6b9b0a2..5ca57afce 100644 --- a/mfr/extensions/tabular/settings.py +++ b/mfr/extensions/tabular/settings.py @@ -12,9 +12,9 @@ LIBS = config.get_object('LIBS', { '.csv': [libs.csv_stdlib], '.tsv': [libs.csv_stdlib], - '.gsheet': [libs.xlsx_xlrd], - '.xlsx': [libs.xlsx_xlrd], - '.xls': [libs.xlsx_xlrd], + '.gsheet': [libs.xlsx], + '.xlsx': [libs.xlsx], + '.xls': [libs.xls], '.dta': [libs.dta_pandas], '.sav': [libs.sav_stdlib], '.mat': [libs.mat_h5py_scipy], diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py index c61c3e400..a068d42b5 100644 --- a/mfr/extensions/tabular/utilities.py +++ b/mfr/extensions/tabular/utilities.py @@ -8,7 +8,7 @@ from tempfile import NamedTemporaryFile from mfr.extensions.tabular import compat -from mfr.core.exceptions import SubprocessError, TooBigToRenderError +from mfr.core.exceptions import SubprocessError, TooBigToRenderError, CorruptedError from mfr.extensions.tabular.settings import (PSPP_CONVERT_BIN, PSPP_CONVERT_TIMEOUT) @@ -143,28 +143,22 @@ def parse_xls(wb, sheets): def parse_xlsx(wb, sheets): for name in wb.sheetnames: ws = wb[name] - max_row = ws.max_row or 0 - max_col = ws.max_column or 0 - verify_size(max_row, max_col, '.xlsx') - - if max_row == 0 or max_col == 0: - sheets[name] = ([], []) - continue - header_row = next(ws.iter_rows(max_row=1, values_only=True), []) fields = fix_headers(header_row) rows = [ dict(zip(fields, row)) for row in ws.iter_rows(min_row=2, - max_row=max_row, - max_col=max_col, + max_row=MAX_SIZE, + max_col=MAX_SIZE, values_only=True) - ] + ] sheets[name] = (header_population(fields), rows) return sheets def verify_size(rows, cols, ext): + if rows is None or cols is None: + raise CorruptedError if rows > MAX_SIZE or cols > MAX_SIZE: raise TooBigToRenderError('Table is too large to render.', ext, nbr_cols=cols, nbr_rows=rows) From 92841af8ad317917320db330a004b70ee4d3b9bf Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 1 Aug 2025 15:56:32 +0300 Subject: [PATCH 2/3] fixed flake8 --- mfr/extensions/tabular/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py index a068d42b5..efd4dad85 100644 --- a/mfr/extensions/tabular/utilities.py +++ b/mfr/extensions/tabular/utilities.py @@ -151,7 +151,7 @@ def parse_xlsx(wb, sheets): max_row=MAX_SIZE, max_col=MAX_SIZE, values_only=True) - ] + ] sheets[name] = (header_population(fields), rows) return sheets From 402f89aa78eada7e6d37b4c3fde1893780b06b9e Mon Sep 17 00:00:00 2001 From: Oleh Paduchak Date: Fri, 1 Aug 2025 18:04:01 +0300 Subject: [PATCH 3/3] unit tests --- tests/extensions/tabular/test_xlsx_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/extensions/tabular/test_xlsx_tools.py b/tests/extensions/tabular/test_xlsx_tools.py index a5e761e98..9956e338d 100644 --- a/tests/extensions/tabular/test_xlsx_tools.py +++ b/tests/extensions/tabular/test_xlsx_tools.py @@ -9,7 +9,7 @@ class TestTabularPandaTools: def test_xlsx_xlrd(self): with open(os.path.join(BASE, 'files', 'test.xlsx'), 'rb') as fp: - sheets = xlrd_tools.xlsx_xlrd(fp) + sheets = xlrd_tools.xlsx(fp) sheet = sheets.popitem()[1] assert sheet[0][0] == {'field': 'one', 'id': 'one', 'name': 'one', 'sortable': True}