diff --git a/modules/invenio-records-rest/invenio_records_rest/sorter.py b/modules/invenio-records-rest/invenio_records_rest/sorter.py index d14595b84d..a06e413be8 100644 --- a/modules/invenio-records-rest/invenio_records_rest/sorter.py +++ b/modules/invenio-records-rest/invenio_records_rest/sorter.py @@ -23,7 +23,7 @@ import pickle import six -from flask import current_app, request +from flask import current_app, request, has_request_context from .config import RECORDS_REST_DEFAULT_SORT @@ -126,12 +126,14 @@ def default_sorter_factory(search, index): :returns: Tuple of (query, URL arguments). """ sort_arg_name = 'sort' - urlfield = request.values.get(sort_arg_name, '', type=str) + urlfield = request.values.get(sort_arg_name, '', type=str) \ + if has_request_context() else '' # Get default sorting if sort is not specified. if not urlfield: # cast to six.text_type to handle unicodes in Python 2 - has_query = request.values.get('q', type=six.text_type) + has_query = request.values.get('q', type=six.text_type) \ + if has_request_context() else None if current_app.config.get('RECORDS_REST_DEFAULT_SORT'): urlfield = current_app.config['RECORDS_REST_DEFAULT_SORT'].get( index, {}).get('query' if has_query else 'noquery', '') diff --git a/modules/invenio-records-rest/tests/test_sorter.py b/modules/invenio-records-rest/tests/test_sorter.py index f9a0244283..41b13d6359 100644 --- a/modules/invenio-records-rest/tests/test_sorter.py +++ b/modules/invenio-records-rest/tests/test_sorter.py @@ -13,6 +13,7 @@ import pytest from elasticsearch_dsl import Search +import invenio_records_rest.sorter as sorter from invenio_records_rest.sorter import default_sorter_factory, eval_field, \ geolocation_sort, parse_sort_field, reverse_order @@ -101,14 +102,16 @@ def test_default_sorter_factory(app): with app.test_request_context("?sort=myfield"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}] + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] assert urlargs['sort'] == 'myfield' # Reverse sort with app.test_request_context("?sort=-myfield"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs['sort'] == '-myfield' # Invalid sort key @@ -121,24 +124,72 @@ def test_default_sorter_factory(app): with app.test_request_context("/?q="): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'asc'}}, {'field2': {'order': 'desc'}}] + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='myfield') # Default sort with query with app.test_request_context("/?q=test"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='-myfield') # Default sort with query that includes unicodes with app.test_request_context("/?q=tést"): query, urlargs = default_sorter_factory(Search(), 'myindex') assert query.to_dict()['sort'] == \ - [{'field1': {'order': 'desc'}}, {'field2': {'order': 'asc'}}] + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] assert urlargs == dict(sort='-myfield') # Default sort another index with app.test_request_context("/?q=test"): query, urlargs = default_sorter_factory(Search(), 'aidx') assert 'sort' not in query.to_dict() + + +def test_default_sorter_factory_without_request_context(app, monkeypatch): + """Test default sorter factory without request context.""" + app.config["RECORDS_REST_SORT_OPTIONS"] = dict( + myindex=dict( + myfield=dict( + fields=['field1', '-field2'], + ) + ), + ) + app.config["RECORDS_REST_DEFAULT_SORT"] = dict( + myindex=dict( + query='-myfield', + noquery='myfield', + ), + ) + + with app.app_context(): + query, urlargs = default_sorter_factory(Search(), 'myindex') + + assert query.to_dict()['sort'] == \ + [{'field1': {'order': 'asc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'desc', 'unmapped_type': 'long'}}] + assert urlargs == dict(sort='myfield') + + app.config["RECORDS_REST_DEFAULT_SORT"] = {} + monkeypatch.setattr( + sorter, + "RECORDS_REST_DEFAULT_SORT", + dict( + myindex=dict( + query='-myfield', + noquery='myfield', + ), + ), + ) + + with app.test_request_context("/?q=test"): + query, urlargs = default_sorter_factory(Search(), 'myindex') + + assert query.to_dict()['sort'] == \ + [{'field1': {'order': 'desc', 'unmapped_type': 'long'}}, + {'field2': {'order': 'asc', 'unmapped_type': 'long'}}] + assert urlargs == dict(sort='-myfield') diff --git a/modules/invenio-stats/invenio_stats/utils.py b/modules/invenio-stats/invenio_stats/utils.py index 3a95d4a6b3..90db8e0151 100644 --- a/modules/invenio-stats/invenio_stats/utils.py +++ b/modules/invenio-stats/invenio_stats/utils.py @@ -1079,7 +1079,7 @@ def get(cls, **kwargs): query_month = str(year) + '-' + str(month).zfill(2) _, lastday = calendar.monthrange(year, month) start_date = query_month + '-01' - end_date = query_month + '-' + str(lastday).zfill(2) + 'T23:59:59' + end_date = query_month + '-' + str(lastday).zfill(2) else: query_month = f"{start_date}-{end_date}" @@ -1089,6 +1089,7 @@ def get(cls, **kwargs): total = 1 count = 0 while count < total and (after_key or first_search): + end_date = end_date + 'T23:59:59' agg_query = cls.build_query(start_date, end_date, after_key) current_app.logger.debug(agg_query.to_dict()) temp_res = agg_query.execute().to_dict() diff --git a/modules/invenio-stats/invenio_stats/utils_search.py b/modules/invenio-stats/invenio_stats/utils_search.py index ffabc584af..9f37dd88ff 100644 --- a/modules/invenio-stats/invenio_stats/utils_search.py +++ b/modules/invenio-stats/invenio_stats/utils_search.py @@ -11,7 +11,7 @@ def billing_file_search_factory(search): from invenio_records_rest.sorter import default_sorter_factory # add Permission filter by publish date and status - mst, _ = get_permission_filter() + mst = get_permission_filter() # billing file search filter query = Q('bool', must=[{'terms': {'content.billing.raw': ['billing_file']}}]) @@ -53,59 +53,22 @@ def get_permission_filter(index_id: str = None): List: Query command. """ - from weko_index_tree.api import Indexes - is_perm = Permission(action_factory("search-access")).can() match = Q("match", publish_status="0") version = Q("match", relation_version_is_last="true") rng = Q("range", **{"publish_date": {"lte": "now/d"}}) - term_list = [] mst = [] - is_perm_paths = Indexes.get_browsing_tree_paths() - is_perm_indexes = [item.split("/")[-1] for item in is_perm_paths] - search_type = request.values.get("search_type") - - if index_id: - index_id = str(index_id) - if search_type == "0": - should_path = [] - if index_id in is_perm_indexes: - should_path.append(Q("terms", path=index_id)) - - mst.append(match) - mst.append(rng) - terms = Q("bool", should=should_path) - else: # In case search_type is keyword or index - if index_id in is_perm_indexes: - term_list.append(index_id) - - mst.append(match) - mst.append(rng) - terms = Q("terms", path=term_list) - else: - mst.append(match) - mst.append(rng) - terms = Q("terms", path=is_perm_indexes) + + mst.append(match) + mst.append(rng) mut = [] + shuld= [] + + shuld.append(Q("bool", must=mst)) + mut.append(Q("bool", should=shuld)) + mut.append(Q("bool", must=version)) - if is_perm: - user_id, result = check_admin_user() - - if result: - shuld = [ - Q("match", weko_creator_id=user_id), - Q("match", weko_shared_id=user_id), - ] - shuld.append(Q("bool", must=mst)) - mut.append(Q("bool", should=shuld, must=[terms])) - mut.append(Q("bool", must=version)) - else: - mut = mst - mut.append(terms) - base_mut = [match, version] - mut.append(Q("bool", must=base_mut)) - - return mut, is_perm_paths + return mut def check_admin_user(): """ diff --git a/modules/invenio-stats/tests/data/billing_file_query.json b/modules/invenio-stats/tests/data/billing_file_query.json index c00f43f088..a0b71c14a5 100644 --- a/modules/invenio-stats/tests/data/billing_file_query.json +++ b/modules/invenio-stats/tests/data/billing_file_query.json @@ -1,6 +1,7 @@ { "_source": { "includes": [ + "path", "content.filename", "_item_metadata.owner", "_oai.id" @@ -14,29 +15,7 @@ "must": [ { "bool": { - "must": [ - { - "terms": { - "path": [ - "1", - "2", - "3", - "4" - ] - } - } - ], "should": [ - { - "match": { - "weko_creator_id": "3" - } - }, - { - "match": { - "weko_shared_id": "3" - } - }, { "bool": { "must": [ diff --git a/modules/invenio-stats/tests/test_utils.py b/modules/invenio-stats/tests/test_utils.py index 5d91620977..e205db96f6 100644 --- a/modules/invenio-stats/tests/test_utils.py +++ b/modules/invenio-stats/tests/test_utils.py @@ -131,7 +131,7 @@ def test_agg_bucket_sort(app): # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_parse_bucket_response -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp def test_parse_bucket_response(app): _raw_res = {'buckets': [{'key': 'test_value'}], 'field': 'test_name'} - + res = parse_bucket_response(_raw_res, {}) assert res=={'test_name': 'test_value'} @@ -145,7 +145,7 @@ def test_get_doctype(app): def test_is_valid_access(app): res = is_valid_access() assert res==True - + with patch("invenio_stats.utils.get_remote_addr", return_value='0.0.0.0'): app.config['STATS_EXCLUDED_ADDRS'] = ['0.0.0.0'] res = is_valid_access() @@ -315,14 +315,14 @@ def test_query_file_reports_helper(i18n_app, roles, mock_es_execute, index): 'get-file-download-per-user-report': None, 'get-file-preview-per-user-report': None}, _data_list) - assert _data_list=={} + assert _data_list=={} QueryFileReportsHelper.Calculation(_report_res, _data_list) assert _data_list=={ 1: {'cur_user_id': 1, 'total_download': 2, 'total_preview': 5}, 2: {'cur_user_id': 2, 'total_download': 3}, 3: {'cur_user_id': 3, 'total_download': 4}, 4: {'cur_user_id': 4, 'total_preview': 1}} - + expect_all_groups = [] def mock_Calculation(res, data_list, all_groups=set(), event=None): if event == 'billing_file_download': @@ -340,7 +340,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): # get_file_stats_report with patch("invenio_stats.utils.QueryFileReportsHelper.Calculation", side_effect=mock_Calculation): with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_res): - res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10) + res = QueryFileReportsHelper.get_file_stats_report(event='file_download', year=2022, month=10) assert res=={ 'all': _expect_data_list, 'all_groups': expect_all_groups, @@ -356,7 +356,7 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): 'open_access': _expect_data_list } with patch("invenio_stats.queries.ESTermsQuery.run", return_value=_billing_res): - res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10) + res = QueryFileReportsHelper.get_file_stats_report(event='billing_file_download', year=2022, month=10) assert res=={ 'all': _expect_billing_data_list, 'all_groups': expect_all_groups, @@ -378,9 +378,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): # get res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user') assert res=={'all': {}, 'date': '2022-10'} res = QueryFileReportsHelper.get(year=2022, month=10, event='test') @@ -398,9 +398,9 @@ def mock_Calculation(res, data_list, all_groups=set(), event=None): def test_query_file_reports_helper_error(app): # get res = QueryFileReportsHelper.get(year=2022, month=10, event='file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='billing_file_download') - assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} + assert res=={'all': [], 'all_groups': [], 'date': '2022-10', 'open_access': []} res = QueryFileReportsHelper.get(year=2022, month=10, event='file_using_per_user') assert res=={'all': {}, 'date': '2022-10'} res = QueryFileReportsHelper.get(year=2022, month=10, event='test') @@ -611,7 +611,7 @@ def test_query_access_counter_helper(app, es): } with patch('invenio_stats.queries.ESTermsQuery.run', return_value=_res): res = QueryAccessCounterHelper.get_top_page_access_counter(year=2022, month=10, start_date='2022-10-01', end_date='2022-10-10') - assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}} + assert res=={'date': '2022-10-01-2022-10-10', 'all':{"count":152}} # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils.py::test_query_access_counter_helper_error -v -s -vv --cov-branch --cov-report=term --cov-config=tox.ini --basetemp=/code/modules/invenio-stats/.tox/c1/tmp def test_query_access_counter_helper_error(app): # get_top_page_access_counter @@ -666,6 +666,26 @@ def test_query_record_view_per_index_report_helper(app, es): res = QueryRecordViewPerIndexReportHelper.get(start_date='2022-10-01', end_date='2022-10-31') assert res=={'all': [], 'date': '2022-10-01-2022-10-31', 'total': 0} + # end_dateに時間が追加されているか + with patch.object(QueryRecordViewPerIndexReportHelper, 'build_query')\ + as mock_build_query: + mock_agg_query = MagicMock() + mock_agg_query.execute.return_value.to_dict.return_value = { + 'aggregations': { + QueryRecordViewPerIndexReportHelper.nested_path: { + 'my_buckets': {'buckets': []}, + 'doc_count': 0 + } + } + } + mock_build_query.return_value = mock_agg_query + with patch.object(QueryRecordViewPerIndexReportHelper, + 'parse_bucket_response', return_value=0): + QueryRecordViewPerIndexReportHelper.get( + start_date='2022-10-01', end_date='2022-10-31') + called_args = mock_build_query.call_args[0] + assert called_args[1] == '2022-10-31T23:59:59' + def test_query_record_view_per_index_report_helper_error(app): # get res = QueryRecordViewPerIndexReportHelper.get(year=2022, month=10) @@ -1086,7 +1106,7 @@ def test_StatsCliUtil(app, db): ) assert not stats_cli.delete_data(True) - + stats_cli = StatsCliUtil( StatsCliUtil.EVENTS_TYPE, _empty_types, verbose=False diff --git a/modules/invenio-stats/tests/test_utils_search.py b/modules/invenio-stats/tests/test_utils_search.py index e13d3ad93e..5bb96bada7 100644 --- a/modules/invenio-stats/tests/test_utils_search.py +++ b/modules/invenio-stats/tests/test_utils_search.py @@ -22,46 +22,40 @@ def test_billing_file_search_factory(i18n_app, role_users, indextree): from os.path import join, dirname search = RecordsSearch() with patch("flask_login.utils._get_user", return_value=role_users[3]['obj']): - with patch('flask_principal.Permission.can', MagicMock(return_value=True)): - search, urlkwargs = billing_file_search_factory(search) - current_app.logger.info(search) - current_app.logger.info(urlkwargs) - with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: - expected = json.load(json_file) - assert search.to_dict() == expected - with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): - with pytest.raises(InvalidQueryRESTError) as e: - billing_file_search_factory(RecordsSearch()) + search, urlkwargs = billing_file_search_factory(search) + current_app.logger.info(search) + current_app.logger.info(urlkwargs) + with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: + expected = json.load(json_file) + assert search.to_dict() == expected + with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): + with pytest.raises(InvalidQueryRESTError) as e: + billing_file_search_factory(RecordsSearch()) + + # auto send report + search2 = RecordsSearch() + search2, urlkwargs2 = billing_file_search_factory(search2) + current_app.logger.info(search2) + current_app.logger.info(urlkwargs2) + with open(join(dirname(__file__), 'data', 'billing_file_query.json')) as json_file: + expected2 = json.load(json_file) + assert search2.to_dict() == expected2 + with patch('invenio_search.api.RecordsSearch.filter', side_effect=SyntaxError()): + with pytest.raises(InvalidQueryRESTError) as e: + billing_file_search_factory(RecordsSearch()) # def get_permission_filter(index_id: str = None): # .tox/c1/bin/pytest --cov=invenio_stats tests/test_utils_search.py::test_get_permission_filter -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/invenio-stats/.tox/c1/tmp -class MockSearchPerm: - def can(self): - return True - def test_get_permission_filter(i18n_app, role_users, indextree): - with patch("flask_login.utils._get_user", return_value=role_users[3]['obj']): - with patch('flask_principal.Permission.can', MagicMock(return_value=True)): - res = get_permission_filter('33') - assert res==([Bool(must=[Terms(path=[])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1','2','3','4']) - with patch('invenio_stats.utils_search.check_admin_user', return_value=('1', False)): - res = get_permission_filter('33') - assert res==([], ['1', '2', '3', '4']) - res = get_permission_filter('1') - assert res==([], ['1', '2', '3', '4']) - with i18n_app.test_request_context(data={'search_type': '0'}): - res = get_permission_filter('33') - assert res==([Bool(must=[Bool()], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1', '2', '3', '4']) - res = get_permission_filter('1') - assert res==([Bool(must=[Bool(should=[Terms(path='1')])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1', '2', '3', '4']) - mock_searchperm = MagicMock(side_effect=MockSearchPerm) - with patch('weko_search_ui.query.search_permission', mock_searchperm): - res = get_permission_filter() - assert res==([Bool(must=[Terms(path=['1', '2', '3', '4'])], should=[Match(weko_creator_id='3'), Match(weko_shared_id='3'), Bool(must=[Match(publish_status='0'), Range(publish_date={'lte': 'now/d'})])]), Bool(must=[Match(relation_version_is_last='true')])], ['1','2','3','4']) - with patch('flask_principal.Permission.can', return_value=False): - res = get_permission_filter('33') - assert res==([Match(publish_status='0'), Range(publish_date={'lte': 'now/d'}), Terms(path=[]), Bool(must=[Match(publish_status='0'), Match(relation_version_is_last='true')])], ['1','2','3','4']) + res = get_permission_filter() + assert res == [ + Bool(should=[Bool(must=[ + Match(publish_status='0'), + Range(publish_date={'lte': 'now/d'}) + ])]), + Bool(must=Match(relation_version_is_last='true')) + ] # def check_admin_user(): diff --git a/modules/weko-records/tests/test_serializers_utils.py b/modules/weko-records/tests/test_serializers_utils.py index 1e4c11ab53..2cc6e563f6 100644 --- a/modules/weko-records/tests/test_serializers_utils.py +++ b/modules/weko-records/tests/test_serializers_utils.py @@ -456,60 +456,196 @@ def test__set_publication_date(app): # def _set_source_identifier(self, fe, item_map, item_metadata): +# .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test__set_source_identifier -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp def test__set_source_identifier(app): sample_copy = copy.deepcopy(sample) fe = MagicMock() fe.prism = MagicMock() - - def issn(item): - return item - - fe.prism.issn = issn + fe.prism.issn = MagicMock() item_map = { "date.@value": "date.@value", "date.@attributes.dateType": "date.@attributes.dateType", - "sourceIdentifier.@value": "sourceIdentifier", - "sourceIdentifier.@attributes.identifierType": "sourceIdentifier.@attributes.identifierType", } item_metadata = { - "sourceIdentifier": "sourceIdentifier", - "@value": "@value", - "sourceIdentifier.@value": "sourceIdentifier", - "sourceIdentifier.@attributes.identifierType": "sourceIdentifier.@attributes.identifierType", + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + } } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + } + } + item_map = { + "date.@value": "date.@value", + "date.@attributes.dateType": "date.@attributes.dateType", + "sourceIdentifier.@value": "item_1.subitem_1", + "sourceIdentifier.@attributes.identifierType": "item_1.subitem_3", + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source1") + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier"] = { - "sourceIdentifier": "sourceIdentifier", + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "ISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + }, + "item_2": { + "attribute_name": "source2", + "attribute_value_mlt": [ + { + "subitem_4": "ISSN", + "subitem_2": "test_source3" + } + ] + } +} + item_map = { + "date.@value": "date.@value", + "date.@attributes.dateType": "date.@attributes.dateType", + "sourceIdentifier.@value": "item_1.subitem_1,item_2.subitem_2", + "sourceIdentifier.@attributes.identifierType": "item_1.subitem_3,item_2.subitem_4", } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source1") + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.assert_any_call("test_source3") + fe.prism.issn.reset_mock() - item_metadata["@value"] = { - "@value": "@value", + item_metadata = { + "item_16": { + "attribute_name": "Title", + "attribute_value_mlt": [ + { + "subitem_15": "test_source", + "subitem_15": "ja" + } + ] + } } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_5": "ISSN", + "subitem_6": "test_source1" + }, + ] + } +} + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier"] = [ - "sourceIdentifier" - ] - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + sample_copy.output_type="rss" - sample_copy.output_type = "not_atom" - item_metadata["sourceIdentifier"] = "ISSN" + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_3": "EISSN", + "subitem_1": "test_source1" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + } + ] + }, + "item_2": { + "attribute_name": "source2", + "attribute_value_mlt": [ + { + "subitem_4": "ISSN", + "subitem_2": "test_source3" + } + ] + } +} + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source2") + fe.prism.issn.assert_any_call("test_source3") + assert fe.prism.issn.call_count == 2 + fe.prism.issn.reset_mock() - with patch("weko_records.serializers.utils.get_metadata_from_map", return_value=item_metadata): - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + "attribute_value_mlt": [ + { + "subitem_1": "test_source1" + }, + { + "subitem_3": "EISSN" + }, + { + "subitem_3": "ISSN", + "subitem_1": "test_source2" + }, + ] + } + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_any_call("test_source2") + assert fe.prism.issn.call_count == 1 + fe.prism.issn.reset_mock() - item_metadata["sourceIdentifier.@attributes.identifierType"] = ["ISSN"] - item_metadata["sourceIdentifier"] = ["ISSN"] + item_metadata = { + "item_1": { + "attribute_name": "Source Identifier", + } + } + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() + fe.prism.issn.reset_mock() - assert sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) == None + with patch("weko_records.serializers.utils.get_metadata_from_map",side_effect=Exception("test error")): + sample_copy._set_source_identifier(fe=fe, item_map=item_map, item_metadata=item_metadata) + fe.prism.issn.assert_not_called() # .tox/c1/bin/pytest --cov=weko_records tests/test_serializers_utils.py::test__set_author_info -v -s -vv --cov-branch --cov-report=term --cov-report=html --cov-config=tox.ini --basetemp=/code/modules/weko-records/.tox/c1/tmp # def _set_author_info(self, fe, item_map, item_metadata, request_lang): diff --git a/modules/weko-records/weko_records/serializers/utils.py b/modules/weko-records/weko_records/serializers/utils.py index cc3a459061..54b2c6e008 100644 --- a/modules/weko-records/weko_records/serializers/utils.py +++ b/modules/weko-records/weko_records/serializers/utils.py @@ -834,33 +834,17 @@ def _set_source_identifier(self, fe, item_map, item_metadata): else: fe.prism.issn(source_identifiers) else: - source_identifiers = source_identifier_metadata.get( - source_identifier_value_key) - source_identifier_types = source_identifier_metadata.get( - source_identifier_attr_type_key) - if source_identifiers and source_identifier_types: - if isinstance(source_identifiers, str): - source_identifiers = [source_identifiers] - if isinstance(source_identifier_types, str): - source_identifier_types = [source_identifier_types] - if len(source_identifiers) != len(source_identifier_types): - attr = item_metadata.get(item_id).get("attribute_value_mlt") - if not attr: - continue - for a in attr: - if a.get(source_identifier_attr_type_key.split('.')[1]) \ - and a.get(source_identifier_value_key.split('.')[1]): - continue - value = next(iter(a.values())) - if value in source_identifiers: - source_identifiers.remove(value) - if value in source_identifier_types: - source_identifier_types.remove(value) - for source_identifier_type, source_identifier in zip( - source_identifier_types, source_identifiers - ): - if source_identifier_type == 'ISSN': - fe.prism.issn(source_identifier) + attr = item_metadata.get(item_id).get("attribute_value_mlt") + if not attr: + continue + type_key = source_identifier_attr_type_key.split('.')[1] + value_key = source_identifier_value_key.split('.')[1] + for a in attr: + source_identifier_type = a.get(type_key) + source_identifier_value = a.get(value_key) + if source_identifier_type and source_identifier_value \ + and source_identifier_type == 'ISSN': + fe.prism.issn(source_identifier_value) except Exception as ex: current_app.logger.error(ex)