From d3b5f44c1043dfcc7f920594bf5b394c1e91fd88 Mon Sep 17 00:00:00 2001 From: ivis-kikuchi Date: Tue, 12 May 2026 09:34:18 +0900 Subject: [PATCH 1/7] add ut --- .../tests/test_serializers_utils.py | 204 +++++++++++++++--- .../weko_records/serializers/utils.py | 2 - 2 files changed, 174 insertions(+), 32 deletions(-) diff --git a/modules/weko-records/tests/test_serializers_utils.py b/modules/weko-records/tests/test_serializers_utils.py index 1e4c11ab53..831c8f1738 100644 --- a/modules/weko-records/tests/test_serializers_utils.py +++ b/modules/weko-records/tests/test_serializers_utils.py @@ -456,60 +456,204 @@ def test__set_publication_date(app): # def _set_source_identifier(self, fe, item_map, item_metadata): +# .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test__set_source_identifier -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp def test__set_source_identifier(app): sample_copy = copy.deepcopy(sample) fe = MagicMock() fe.prism = MagicMock() - - def issn(item): - return item - - fe.prism.issn = issn + fe.prism.issn = MagicMock() item_map = { "date.@value": "date.@value", "date.@attributes.dateType": "date.@attributes.dateType", - "sourceIdentifier.@value": "sourceIdentifier", - "sourceIdentifier.@attributes.identifierType": "sourceIdentifier.@attributes.identifierType", } item_metadata = { - "sourceIdentifier": "sourceIdentifier", - "@value": "@value", - "sourceIdentifier.@value": "sourceIdentifier", - "sourceIdentifier.@attributes.identifierType": "sourceIdentifier.@attributes.identifierType", + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + } } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + } + } + item_map = { + "date.@value": "date.@value", + "date.@attributes.dateType": "date.@attributes.dateType", + "sourceIdentifier.@value": "item_1.subitem_1", + "sourceIdentifier.@attributes.identifierType": "item_1.subitem_3", + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source1") + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier"] = { - "sourceIdentifier": "sourceIdentifier", + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + }, + "item_2": { + "attribute_name": "source2", + "attribute_value_mlt": [ + { + "subitem_4": "ISSN", + "subitem_2": "test_source3" + } + ] + } +} + item_map = { + "date.@value": "date.@value", + "date.@attributes.dateType": "date.@attributes.dateType", + "sourceIdentifier.@value": "item_1.subitem_1,item_2.subitem_2", + "sourceIdentifier.@attributes.identifierType": "item_1.subitem_3,item_2.subitem_4", } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source1") + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.assert_any_call("test_source3") + fe.prism.issn.reset_mock() - item_metadata["@value"] = { - "@value": "@value", + item_metadata = { + "item_16": { + "attribute_name": "Title", + "attribute_value_mlt": [ + { + "subitem_15": "test_source", + "subitem_15": "ja" + } + ] + } } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_5": "ISSN", + "subitem_6": "test_source1" + }, + ] + } +} + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier"] = [ - "sourceIdentifier" - ] - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + sample_copy.output_type="rss" - sample_copy.output_type = "not_atom" - item_metadata["sourceIdentifier"] = "ISSN" + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "EISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + }, + "item_2": { + "attribute_name": "source2", + "attribute_value_mlt": [ + { + "subitem_4": "ISSN", + "subitem_2": "test_source3" + } + ] + } +} + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.assert_any_call("test_source3") + assert fe.prism.issn.call_count == 2 + fe.prism.issn.reset_mock() - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_1": "test_source1" + }, + { + "subitem_3": "EISSN" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + }, + { + "subitem_1": "test_source3" + }, + ] + } + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source2") + assert fe.prism.issn.call_count == 1 + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier.@attributes.identifierType"] = ["ISSN"] - item_metadata["sourceIdentifier"] = ["ISSN"] + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_1": "test" + } + ] + } + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + with patch("weko_records.serializers.utils.get_metadata_from_map",side_effect=Exception("test error")): + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() # .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test__set_author_info -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp # def _set_author_info(self, fe, item_map, item_metadata, request_lang): diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index cc3a459061..e632bee04f 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -845,8 +845,6 @@ def _set_source_identifier(self, fe, item_map, item_metadata): source_identifier_types = [source_identifier_types] if len(source_identifiers) != len(source_identifier_types): attr = item_metadata.get(item_id).get("attribute_value_mlt") - if not attr: - continue for a in attr: if a.get(source_identifier_attr_type_key.split('.')[1]) \ and a.get(source_identifier_value_key.split('.')[1]): From 0a1879eecf8f9a8b5ff52093ccec312f85ea67de Mon Sep 17 00:00:00 2001 From: ivis-kikuchi Date: Tue, 12 May 2026 13:25:01 +0900 Subject: [PATCH 2/7] fix --- .../weko_records/serializers/utils.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index e632bee04f..3e4cc1863a 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -843,17 +843,14 @@ def _set_source_identifier(self, fe, item_map, item_metadata): source_identifiers = [source_identifiers] if isinstance(source_identifier_types, str): source_identifier_types = [source_identifier_types] - if len(source_identifiers) != len(source_identifier_types): - attr = item_metadata.get(item_id).get("attribute_value_mlt") - for a in attr: - if a.get(source_identifier_attr_type_key.split('.')[1]) \ - and a.get(source_identifier_value_key.split('.')[1]): - continue - value = next(iter(a.values())) - if value in source_identifiers: - source_identifiers.remove(value) - if value in source_identifier_types: - source_identifier_types.remove(value) + source_identifiers.clear() + source_identifier_types.clear() + attr = item_metadata.get(item_id).get("attribute_value_mlt") + for a in attr: + if a.get(source_identifier_attr_type_key.split('.')[1]) \ + and a.get(source_identifier_value_key.split('.')[1]): + source_identifier_types.append(a.get(source_identifier_attr_type_key.split('.')[1])) + source_identifiers.append(a.get(source_identifier_value_key.split('.')[1])) for source_identifier_type, source_identifier in zip( source_identifier_types, source_identifiers ): From 38354fa530669dbb04fd9bb04dcf3481b0198106 Mon Sep 17 00:00:00 2001 From: ivis-kikuchi Date: Tue, 12 May 2026 16:45:47 +0900 Subject: [PATCH 3/7] fix --- .../weko_records/serializers/utils.py | 37 ++++++++----------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index 3e4cc1863a..74a143fbee 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -834,28 +834,21 @@ def _set_source_identifier(self, fe, item_map, item_metadata): else: fe.prism.issn(source_identifiers) else: - source_identifiers = source_identifier_metadata.get( - source_identifier_value_key) - source_identifier_types = source_identifier_metadata.get( - source_identifier_attr_type_key) - if source_identifiers and source_identifier_types: - if isinstance(source_identifiers, str): - source_identifiers = [source_identifiers] - if isinstance(source_identifier_types, str): - source_identifier_types = [source_identifier_types] - source_identifiers.clear() - source_identifier_types.clear() - attr = item_metadata.get(item_id).get("attribute_value_mlt") - for a in attr: - if a.get(source_identifier_attr_type_key.split('.')[1]) \ - and a.get(source_identifier_value_key.split('.')[1]): - source_identifier_types.append(a.get(source_identifier_attr_type_key.split('.')[1])) - source_identifiers.append(a.get(source_identifier_value_key.split('.')[1])) - for source_identifier_type, source_identifier in zip( - source_identifier_types, source_identifiers - ): - if source_identifier_type == 'ISSN': - fe.prism.issn(source_identifier) + source_identifiers = [] + source_identifier_types = [] + attr = item_metadata.get(item_id).get("attribute_value_mlt") + if not attr: + continue + for a in attr: + if a.get(source_identifier_attr_type_key.split('.')[1]) \ + and a.get(source_identifier_value_key.split('.')[1]): + source_identifier_types.append(a.get(source_identifier_attr_type_key.split('.')[1])) + source_identifiers.append(a.get(source_identifier_value_key.split('.')[1])) + for source_identifier_type, source_identifier in zip( + source_identifier_types, source_identifiers + ): + if source_identifier_type == 'ISSN': + fe.prism.issn(source_identifier) except Exception as ex: current_app.logger.error(ex) From 18f2ab8d776856f7fee06157c3b2d9a7b9c969f9 Mon Sep 17 00:00:00 2001 From: ivis-kikuchi Date: Tue, 12 May 2026 17:32:17 +0900 Subject: [PATCH 4/7] fix --- .../weko_records/serializers/utils.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index 74a143fbee..54b2c6e008 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -834,21 +834,17 @@ def _set_source_identifier(self, fe, item_map, item_metadata): else: fe.prism.issn(source_identifiers) else: - source_identifiers = [] - source_identifier_types = [] attr = item_metadata.get(item_id).get("attribute_value_mlt") if not attr: continue + type_key = source_identifier_attr_type_key.split('.')[1] + value_key = source_identifier_value_key.split('.')[1] for a in attr: - if a.get(source_identifier_attr_type_key.split('.')[1]) \ - and a.get(source_identifier_value_key.split('.')[1]): - source_identifier_types.append(a.get(source_identifier_attr_type_key.split('.')[1])) - source_identifiers.append(a.get(source_identifier_value_key.split('.')[1])) - for source_identifier_type, source_identifier in zip( - source_identifier_types, source_identifiers - ): - if source_identifier_type == 'ISSN': - fe.prism.issn(source_identifier) + source_identifier_type = a.get(type_key) + source_identifier_value = a.get(value_key) + if source_identifier_type and source_identifier_value \ + and source_identifier_type == 'ISSN': + fe.prism.issn(source_identifier_value) except Exception as ex: current_app.logger.error(ex) From 062052df1fc90b380feeb60a89afe402ab88e33d Mon Sep 17 00:00:00 2001 From: ivis-kikuchi Date: Tue, 12 May 2026 17:47:39 +0900 Subject: [PATCH 5/7] fix --- modules/weko-records/tests/test_serializers_utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/modules/weko-records/tests/test_serializers_utils.py b/modules/weko-records/tests/test_serializers_utils.py index 831c8f1738..2cc6e563f6 100644 --- a/modules/weko-records/tests/test_serializers_utils.py +++ b/modules/weko-records/tests/test_serializers_utils.py @@ -626,9 +626,6 @@ def test__set_source_identifier(app): "subitem_3": "ISSN", "subitem_1": "test_source2" }, - { - "subitem_1": "test_source3" - }, ] } } @@ -640,11 +637,6 @@ def test__set_source_identifier(app): item_metadata = { "item_1": { "attribute_name": "Source Identifier", - "attribute_value_mlt": [ - { - "subitem_1": "test" - } - ] } } sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) From 0224d4f8cd02a5c49a4ccfe42b1225ff0d8317ef Mon Sep 17 00:00:00 2001 From: ayumi-nishida Date: Wed, 13 May 2026 15:42:31 +0900 Subject: [PATCH 6/7] =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=83=AC=E3=83=9D?= =?UTF-8?q?=E3=83=BC=E3=83=88=E9=9B=86=E8=A8=88=E5=AF=BE=E8=B1=A1=E3=81=AE?= =?UTF-8?q?end=5Ftime=E3=81=AB"T23:59:59"=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- modules/invenio-stats/invenio_stats/utils.py | 3 +- modules/invenio-stats/tests/test_utils.py | 44 ++++++++++++++------ 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/modules/invenio-stats/invenio_stats/utils.py b/modules/invenio-stats/invenio_stats/utils.py index 3a95d4a6b3..90db8e0151 100644 --- a/modules/invenio-stats/invenio_stats/utils.py +++ b/modules/invenio-stats/invenio_stats/utils.py @@ -1079,7 +1079,7 @@ def get(cls, **kwargs): query_month = str(year) + '-' + str(month).zfill(2) _, lastday = calendar.monthrange(year, month) start_date = query_month + '-01' - end_date = query_month + '-' + str(lastday).zfill(2) + 'T23:59:59' + end_date = query_month + '-' + str(lastday).zfill(2) else: query_month = f"{start_date}-{end_date}" @@ -1089,6 +1089,7 @@ def get(cls, **kwargs): total = 1 count = 0 while count < total and (after_key or first_search): + end_date = end_date + 'T23:59:59' agg_query = cls.build_query(start_date, end_date, after_key) current_app.logger.debug(agg_query.to_dict()) temp_res = agg_query.execute().to_dict() diff --git a/modules/invenio-stats/tests/test_utils.py b/modules/invenio-stats/tests/test_utils.py index 5d91620977..e205db96f6 100644 --- a/modules/invenio-stats/tests/test_utils.py +++ b/modules/invenio-stats/tests/test_utils.py @@ -131,7 +131,7 @@ def test_agg_bucket_sort(app): # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_parse_bucket_response -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp def test_parse_bucket_response(app): _raw_res = {'buckets': [{'key': 'test_value'}], 'field': 'test_name'} - + res = parse_bucket_response(_raw_res, {}) assert res=={'test_name': 'test_value'} @@ -145,7 +145,7 @@ def test_get_doctype(app): def test_is_valid_access(app): res = is_valid_access() assert res==True - + with patch("invenio_stats.utils.get_remote_addr", return_value='0.0.0.0'): app.config['STATS_EXCLUDED_ADDRS'] = ['0.0.0.0'] res = is_valid_access() @@ -315,14 +315,14 @@ def test_query_file_reports_helper(i18n_app, roles, mock_es_execute, index): 'get-file-download-per-user-report': None, 'get-file-preview-per-user-report': None}, _data_list) - assert _data_list=={} + assert _data_list=={} QueryFileReportsHelper.Calculation(_report_res, _data_list) assert _data_list=={ 1: {'cur_user_id': 1, 'total_download': 2, 'total_preview': 5}, 2: {'cur_user_id': 2, 'total_download': 3}, 3: {'cur_user_id': 3, 'total_download': 4}, 4: {'cur_user_id': 4, 'total_preview': 1}} - + expect_all_groups = [] def mock_Calculation(res, data_list, all_groups=set(), event=None): if event == 'billing_file_download': @@ -340,7 +340,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): # get_file_stats_report with patch("invenio_stats.utils.QueryFileReportsHelper.Calculation", side_effect=mock_Calculation): with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_res): - res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10) + res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10) assert res=={ 'all': _expect_data_list, 'all_groups': expect_all_groups, @@ -356,7 +356,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): 'open_access': _expect_data_list } with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_billing_res): - res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10) + res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10) assert res=={ 'all': _expect_billing_data_list, 'all_groups': expect_all_groups, @@ -378,9 +378,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): # get res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user') assert res=={'all': {}, 'date': '2022-10'} res = QueryFileReportsHelper.get(year=2022, month=10, event='test') @@ -398,9 +398,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): def test_query_file_reports_helper_error(app): # get res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user') assert res=={'all': {}, 'date': '2022-10'} res = QueryFileReportsHelper.get(year=2022, month=10, event='test') @@ -611,7 +611,7 @@ def test_query_access_counter_helper(app, es): } with patch('invenio_stats.queries.ESTermsQuery.run', return_value=_res): res = QueryAccessCounterHelper.get_top_page_access_counter(year=2022, month=10, start_date='2022-10-01', end_date='2022-10-10') - assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}} + assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}} # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_query_access_counter_helper_error -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp def test_query_access_counter_helper_error(app): # get_top_page_access_counter @@ -666,6 +666,26 @@ def test_query_record_view_per_index_report_helper(app, es): res = QueryRecordViewPerIndexReportHelper.get(start_date='2022-10-01', end_date='2022-10-31') assert res=={'all': [], 'date': '2022-10-01-2022-10-31', 'total': 0} + # end_dateに時間が追加されているか + with patch.object(QueryRecordViewPerIndexReportHelper, 'build_query')\ + as mock_build_query: + mock_agg_query = MagicMock() + mock_agg_query.execute.return_value.to_dict.return_value = { + 'aggregations': { + QueryRecordViewPerIndexReportHelper.nested_path: { + 'my_buckets': {'buckets': []}, + 'doc_count': 0 + } + } + } + mock_build_query.return_value = mock_agg_query + with patch.object(QueryRecordViewPerIndexReportHelper, + 'parse_bucket_response', return_value=0): + QueryRecordViewPerIndexReportHelper.get( + start_date='2022-10-01', end_date='2022-10-31') + called_args = mock_build_query.call_args[0] + assert called_args[1] == '2022-10-31T23:59:59' + def test_query_record_view_per_index_report_helper_error(app): # get res = QueryRecordViewPerIndexReportHelper.get(year=2022, month=10) @@ -1086,7 +1106,7 @@ def test_StatsCliUtil(app, db): ) assert not stats_cli.delete_data(True) - + stats_cli = StatsCliUtil( StatsCliUtil.EVENTS_TYPE, _empty_types, verbose=False From 6d8d3f8e3634120d28bab317fb77b494b7c712ce Mon Sep 17 00:00:00 2001 From: kanon-yoneji Date: Thu, 14 May 2026 10:15:29 +0900 Subject: [PATCH 7/7] fix payFileDownload report list query for autosend --- .../invenio_records_rest/sorter.py | 8 ++- .../invenio-records-rest/tests/test_sorter.py | 61 ++++++++++++++++-- .../invenio_stats/utils_search.py | 57 +++-------------- .../tests/data/billing_file_query.json | 23 +------ .../invenio-stats/tests/test_utils_search.py | 64 +++++++++---------- 5 files changed, 101 insertions(+), 112 deletions(-) diff --git a/modules/invenio-records-rest/invenio_records_rest/sorter.py b/modules/invenio-records-rest/invenio_records_rest/sorter.py index d14595b84d..a06e413be8 100644 --- a/modules/invenio-records-rest/invenio_records_rest/sorter.py +++ b/modules/invenio-records-rest/invenio_records_rest/sorter.py @@ -23,7 +23,7 @@ import pickle import six -from flask import current_app, request +from flask import current_app, request, has_request_context from .config import RECORDS_REST_DEFAULT_SORT @@ -126,12 +126,14 @@ def default_sorter_factory(search, index): :returns: Tuple of (query, URL arguments). """ sort_arg_name = 'sort' - urlfield = request.values.get(sort_arg_name, '', type=str) + urlfield = request.values.get(sort_arg_name, '', type=str) \ + if has_request_context() else '' # Get default sorting if sort is not specified. if not urlfield: # cast to six.text_type to handle unicodes in Python 2 - has_query = request.values.get('q', type=six.text_type) + has_query = request.values.get('q', type=six.text_type) \ + if has_request_context() else None if current_app.config.get('RECORDS_REST_DEFAULT_SORT'): urlfield = current_app.config['RECORDS_REST_DEFAULT_SORT'].get( index, {}).get('query' if has_query else 'noquery', '') diff --git a/modules/invenio-records-rest/tests/test_sorter.py b/modules/invenio-records-rest/tests/test_sorter.py index f9a0244283..41b13d6359 100644 --- a/modules/invenio-records-rest/tests/test_sorter.py +++ b/modules/invenio-records-rest/tests/test_sorter.py @@ -13,6 +13,7 @@ import pytest from elasticsearch_dsl import Search +import invenio_records_rest.sorter as sorter from invenio_records_rest.sorter import default_sorter_factory, eval_field, \ geolocation_sort, parse_sort_field, reverse_order @@ -101,14 +102,16 @@ def test_default_sorter_factory(app): with app.test_request_context("?sort=myfield"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}] + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] assert urlargs['sort'] == 'myfield' # Reverse sort with app.test_request_context("?sort=-myfield"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs['sort'] == '-myfield' # Invalid sort key @@ -121,24 +124,72 @@ def test_default_sorter_factory(app): with app.test_request_context("/?q="): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}] + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='myfield') # Default sort with query with app.test_request_context("/?q=test"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='-myfield') # Default sort with query that includes unicodes with app.test_request_context("/?q=tést"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='-myfield') # Default sort another index with app.test_request_context("/?q=test"): query, urlargs = default_sorter_factory(Search(), 'aidx') assert 'sort' not in query.to_dict() + + +def test_default_sorter_factory_without_request_context(app, monkeypatch): + """Test default sorter factory without request context.""" + app.config["RECORDS_REST_SORT_OPTIONS"] = dict( + myindex=dict( + myfield=dict( + fields=['field1', '-field2'], + ) + ), + ) + app.config["RECORDS_REST_DEFAULT_SORT"] = dict( + myindex=dict( + query='-myfield', + noquery='myfield', + ), + ) + + with app.app_context(): + query, urlargs = default_sorter_factory(Search(), 'myindex') + + assert query.to_dict()['sort'] == \ + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] + assert urlargs == dict(sort='myfield') + + app.config["RECORDS_REST_DEFAULT_SORT"] = {} + monkeypatch.setattr( + sorter, + "RECORDS_REST_DEFAULT_SORT", + dict( + myindex=dict( + query='-myfield', + noquery='myfield', + ), + ), + ) + + with app.test_request_context("/?q=test"): + query, urlargs = default_sorter_factory(Search(), 'myindex') + + assert query.to_dict()['sort'] == \ + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] + assert urlargs == dict(sort='-myfield') diff --git a/modules/invenio-stats/invenio_stats/utils_search.py b/modules/invenio-stats/invenio_stats/utils_search.py index ffabc584af..9f37dd88ff 100644 --- a/modules/invenio-stats/invenio_stats/utils_search.py +++ b/modules/invenio-stats/invenio_stats/utils_search.py @@ -11,7 +11,7 @@ def billing_file_search_factory(search): from invenio_records_rest.sorter import default_sorter_factory # add Permission filter by publish date and status - mst, _ = get_permission_filter() + mst = get_permission_filter() # billing file search filter query = Q('bool', must=[{'terms': {'content.billing.raw': ['billing_file']}}]) @@ -53,59 +53,22 @@ def get_permission_filter(index_id: str = None): List: Query command. """ - from weko_index_tree.api import Indexes - is_perm = Permission(action_factory("search-access")).can() match = Q("match", publish_status="0") version = Q("match", relation_version_is_last="true") rng = Q("range", **{"publish_date": {"lte": "now/d"}}) - term_list = [] mst = [] - is_perm_paths = Indexes.get_browsing_tree_paths() - is_perm_indexes = [item.split("/")[-1] for item in is_perm_paths] - search_type = request.values.get("search_type") - - if index_id: - index_id = str(index_id) - if search_type == "0": - should_path = [] - if index_id in is_perm_indexes: - should_path.append(Q("terms", path=index_id)) - - mst.append(match) - mst.append(rng) - terms = Q("bool", should=should_path) - else: # In case search_type is keyword or index - if index_id in is_perm_indexes: - term_list.append(index_id) - - mst.append(match) - mst.append(rng) - terms = Q("terms", path=term_list) - else: - mst.append(match) - mst.append(rng) - terms = Q("terms", path=is_perm_indexes) + + mst.append(match) + mst.append(rng) mut = [] + shuld= [] + + shuld.append(Q("bool", must=mst)) + mut.append(Q("bool", should=shuld)) + mut.append(Q("bool", must=version)) - if is_perm: - user_id, result = check_admin_user() - - if result: - shuld = [ - Q("match", weko_creator_id=user_id), - Q("match", weko_shared_id=user_id), - ] - shuld.append(Q("bool", must=mst)) - mut.append(Q("bool", should=shuld, must=[terms])) - mut.append(Q("bool", must=version)) - else: - mut = mst - mut.append(terms) - base_mut = [match, version] - mut.append(Q("bool", must=base_mut)) - - return mut, is_perm_paths + return mut def check_admin_user(): """ diff --git a/modules/invenio-stats/tests/data/billing_file_query.json b/modules/invenio-stats/tests/data/billing_file_query.json index c00f43f088..a0b71c14a5 100644 --- a/modules/invenio-stats/tests/data/billing_file_query.json +++ b/modules/invenio-stats/tests/data/billing_file_query.json @@ -1,6 +1,7 @@ { "_source": { "includes": [ + "path", "content.filename", "_item_metadata.owner", "_oai.id" @@ -14,29 +15,7 @@ "must": [ { "bool": { - "must": [ - { - "terms": { - "path": [ - "1", - "2", - "3", - "4" - ] - } - } - ], "should": [ - { - "match": { - "weko_creator_id": "3" - } - }, - { - "match": { - "weko_shared_id": "3" - } - }, { "bool": { "must": [ diff --git a/modules/invenio-stats/tests/test_utils_search.py b/modules/invenio-stats/tests/test_utils_search.py index e13d3ad93e..5bb96bada7 100644 --- a/modules/invenio-stats/tests/test_utils_search.py +++ b/modules/invenio-stats/tests/test_utils_search.py @@ -22,46 +22,40 @@ def test_billing_file_search_factory(i18n_app, role_users, indextree): from os.path import join, dirname search = RecordsSearch() with patch("flask_login.utils._get_user", return_value=role_users[3]['obj']): - with patch('flask_principal.Permission.can', MagicMock(return_value=True)): - search, urlkwargs = billing_file_search_factory(search) - current_app.logger.info(search) - current_app.logger.info(urlkwargs) - with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: - expected = json.load(json_file) - assert search.to_dict() == expected - with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): - with pytest.raises(InvalidQueryRESTError) as e: - billing_file_search_factory(RecordsSearch()) + search, urlkwargs = billing_file_search_factory(search) + current_app.logger.info(search) + current_app.logger.info(urlkwargs) + with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: + expected = json.load(json_file) + assert search.to_dict() == expected + with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): + with pytest.raises(InvalidQueryRESTError) as e: + billing_file_search_factory(RecordsSearch()) + + # auto send report + search2 = RecordsSearch() + search2, urlkwargs2 = billing_file_search_factory(search2) + current_app.logger.info(search2) + current_app.logger.info(urlkwargs2) + with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: + expected2 = json.load(json_file) + assert search2.to_dict() == expected2 + with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): + with pytest.raises(InvalidQueryRESTError) as e: + billing_file_search_factory(RecordsSearch()) # def get_permission_filter(index_id: str = None): # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils_search.py::test_get_permission_filter -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-stats/.tox/c1/tmp -class MockSearchPerm: - def can(self): - return True - def test_get_permission_filter(i18n_app, role_users, indextree): - with patch("flask_login.utils._get_user", return_value=role_users[3]['obj']): - with patch('flask_principal.Permission.can', MagicMock(return_value=True)): - res = get_permission_filter('33') - assert res==([Bool(must=[Terms(path=[])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1','2','3','4']) - with patch('invenio_stats.utils_search.check_admin_user', return_value=('1', False)): - res = get_permission_filter('33') - assert res==([], ['1', '2', '3', '4']) - res = get_permission_filter('1') - assert res==([], ['1', '2', '3', '4']) - with i18n_app.test_request_context(data={'search_type': '0'}): - res = get_permission_filter('33') - assert res==([Bool(must=[Bool()], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1', '2', '3', '4']) - res = get_permission_filter('1') - assert res==([Bool(must=[Bool(should=[Terms(path='1')])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1', '2', '3', '4']) - mock_searchperm = MagicMock(side_effect=MockSearchPerm) - with patch('weko_search_ui.query.search_permission', mock_searchperm): - res = get_permission_filter() - assert res==([Bool(must=[Terms(path=['1', '2', '3', '4'])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1','2','3','4']) - with patch('flask_principal.Permission.can', return_value=False): - res = get_permission_filter('33') - assert res==([Match(publish_status='0'), Range(publish_date={'lte': 'now/d'}), Terms(path=[]), Bool(must=[Match(publish_status='0'), Match(relation_version_is_last='true')])], ['1','2','3','4']) + res = get_permission_filter() + assert res == [ + Bool(should=[Bool(must=[ + Match(publish_status='0'), + Range(publish_date={'lte': 'now/d'}) + ])]), + Bool(must=Match(relation_version_is_last='true')) + ] # def check_admin_user():