Skip to content

MODINV-1395 - Importing Records from BNE z39.50 - record not available error #925

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RuslanLavrov merged 4 commits into from
Jul 3, 2026
Merged

MODINV-1395 - Importing Records from BNE z39.50 - record not available error #925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1d9526e
Fixed extraction and check for 999ff$i field existence in a record
RuslanLavrov Jul 1, 2026
46ba631
Added tests
RuslanLavrov Jul 1, 2026
a1cbbef
Refactoring
RuslanLavrov Jul 1, 2026
fe24f21
news
RuslanLavrov Jul 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions NEWS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
## 22.1.0-SNAPSHOT 2026-mm-dd
* Data Import profiles will map invalid statistical codes [MODINV-1391](https://folio-org.atlassian.net/browse/MODINV-1391)
* Importing Records from BNE z39.50 - record not available error [MODINV-1395](https://folio-org.atlassian.net/browse/MODINV-1395)

## 22.0.0 2026-04-16
* ECS: Improve logs in move holdings operation for [MODINV-1276](https://folio-org.atlassian.net/browse/MODINV-1276)
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/org/folio/inventory/dataimport/consumers/MarcBibUpdateKafkaHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.OKAPI_REQUEST_ID;
import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.OKAPI_USER_ID;
import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.constructContext;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.INDICATOR_F;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.SUBFIELD_I;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_999;
import static org.folio.inventory.dataimport.util.MappingConstants.MARC_BIB_RECORD_TYPE;
Expand Down Expand Up @@ -212,8 +213,9 @@ private KafkaProducer<String, String> createProducer(String eventType, KafkaConf
}

private LinkUpdateReport mapToLinkReport(MarcBibUpdate marcBibUpdate, String errMessage) {
var instanceId = AdditionalFieldsUtil.getValue(marcBibUpdate.getRecord(), TAG_999, SUBFIELD_I)
.orElse(null);
var instanceId =
AdditionalFieldsUtil.getValueFromDataField(marcBibUpdate.getRecord(), TAG_999, INDICATOR_F, INDICATOR_F, SUBFIELD_I)
.orElse(null);
return new LinkUpdateReport()
.withJobId(marcBibUpdate.getJobId())
.withInstanceId(instanceId)
Expand Down
9 changes: 7 additions & 2 deletions ...main/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static org.folio.DataImportEventTypes.DI_INVENTORY_INSTANCE_CREATED_READY_FOR_POST_PROCESSING;
import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.OKAPI_REQUEST_ID;
import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.PAYLOAD_USER_ID;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.INDICATOR_F;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.SUBFIELD_I;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_999;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.reorderMarcRecordFields;
Expand Down Expand Up @@ -61,7 +62,7 @@ public class CreateInstanceEventHandler extends AbstractInstanceEventHandler {
private static final String PAYLOAD_HAS_NO_DATA_MSG = "Failed to handle event payload - event payload context does not contain MARC_BIBLIOGRAPHIC data";
static final String ACTION_HAS_NO_MAPPING_MSG = "Action profile to create an Instance requires a mapping profile by jobExecutionId: '%s' and recordId: '%s'";
private static final String MAPPING_PARAMETERS_NOT_FOUND_MSG = "MappingParameters snapshot was not found by jobExecutionId: '%s'. RecordId: '%s', chunkId: '%s' ";
private static final String INSTANCE_CREATION_999_ERROR_MESSAGE = "A new Instance was not created because the incoming record already contained a 999ff$s or 999ff$i field";
private static final String INSTANCE_CREATION_999_ERROR_MESSAGE = "A new Instance was not created because the incoming record already contains a 999ff$s or 999ff$i field";
private static final String RECORD_ID_HEADER = "recordId";
private static final String CHUNK_ID_HEADER = "chunkId";
public static final String DISCOVERY_SUPPRESS_PROPERTY = "discoverySuppress";
Expand Down Expand Up @@ -102,7 +103,7 @@ public CompletableFuture<DataImportEventPayload> handle(DataImportEventPayload d
Record targetRecord = Json.decodeValue(payloadContext.get(EntityType.MARC_BIBLIOGRAPHIC.value()), Record.class);
var sourceContent = targetRecord.getParsedRecord().getContent().toString();

if (!Boolean.parseBoolean(payloadContext.get("acceptInstanceId")) && AdditionalFieldsUtil.getValue(targetRecord, TAG_999, SUBFIELD_I).isPresent()) {
if (!Boolean.parseBoolean(payloadContext.get("acceptInstanceId")) && contains999ffSubfieldIValue(targetRecord)) {
LOGGER.error("handle:: {} jobExecutionId: {} recordId: {} ", INSTANCE_CREATION_999_ERROR_MESSAGE, jobExecutionId, recordId);
return CompletableFuture.failedFuture(new EventProcessingException(INSTANCE_CREATION_999_ERROR_MESSAGE));
}
Expand Down Expand Up @@ -183,6 +184,10 @@ public CompletableFuture<DataImportEventPayload> handle(DataImportEventPayload d
return future;
}

private boolean contains999ffSubfieldIValue(Record targetRecord) {
return AdditionalFieldsUtil.getValueFromDataField(targetRecord, TAG_999, INDICATOR_F, INDICATOR_F, SUBFIELD_I).isPresent();
}

private JsonObject prepareInstance(DataImportEventPayload dataImportEventPayload, String instanceId, String jobExecutionId) {
JsonObject instanceAsJson = new JsonObject(dataImportEventPayload.getContext().get(INSTANCE.value()));
if (instanceAsJson.getJsonObject(INSTANCE_PATH) != null) {
Expand Down
61 changes: 38 additions & 23 deletions src/main/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.marc4j.marc.MarcFactory;
import org.marc4j.marc.Subfield;
import org.marc4j.marc.VariableField;
import org.marc4j.marc.impl.Verifier;

/**
* Util to work with additional fields
Expand All @@ -68,7 +69,7 @@ public final class AdditionalFieldsUtil {
public static final char TAG_035_SUB = 'a';
private static final char TAG_035_IND = ' ';
private static final String ANY_STRING = "*";
private static final char INDICATOR = 'f';
public static final char INDICATOR_F = 'f';
public static final char SUBFIELD_I = 'i';
public static final char SUBFIELD_L = 'l';
private static final String HR_ID_FIELD = "hrid";
Expand All @@ -77,6 +78,7 @@ public final class AdditionalFieldsUtil {
private static final String OCLC_PREFIX = "(OCoLC)";
private static final ObjectMapper objectMapper = new ObjectMapper();
public static final String FIELDS = "fields";
static final String INVALID_DATA_FIELD_MSG = "Field '%s' is not a data field.";

static {
// this function is executed when creating a new item to be saved in the cache.
Expand Down Expand Up @@ -150,14 +152,14 @@ public static boolean addFieldToMarcRecord(Record recordForUpdate, String field,
VariableField variableField = getSingleFieldByIndicators(marcRecord.getVariableFields(field));
DataField dataField;
if (variableField != null
&& ((DataField) variableField).getIndicator1() == INDICATOR
&& ((DataField) variableField).getIndicator2() == INDICATOR
&& ((DataField) variableField).getIndicator1() == INDICATOR_F
&& ((DataField) variableField).getIndicator2() == INDICATOR_F
) {
dataField = (DataField) variableField;
marcRecord.removeVariableField(variableField);
dataField.removeSubfield(dataField.getSubfield(subfield));
} else {
dataField = factory.newDataField(field, INDICATOR, INDICATOR);
dataField = factory.newDataField(field, INDICATOR_F, INDICATOR_F);
}
dataField.addSubfield(factory.newSubfield(subfield, value));
marcRecord.addVariableField(dataField);
Expand Down Expand Up @@ -338,22 +340,35 @@ public static String getValueFromControlledField(Record srcRecord, String tag) {
return null;
}

public static Optional<String> getValue(Record srcRecord, String tag, char subfield) {
return Optional.ofNullable(computeMarcRecord(srcRecord))
.stream()
.flatMap(marcRecord -> marcRecord.getVariableFields(tag).stream())
.flatMap(field -> getFieldValue(field, subfield).stream())
.findFirst();
}

private static Optional<String> getFieldValue(VariableField field, char subfield) {
if (field instanceof DataField dataField) {
return dataField.getSubfields(subfield).stream().findFirst().map(Subfield::getData);
} else if (field instanceof ControlField controlField) {
return Optional.ofNullable(controlField.getData());
} else {
return Optional.empty();
/**
* Reads the value of a subfield from the first matching data field in a
* MARC record, identified by {@code tag} and indicator values.
*
* @param srcRecord record containing the parsed MARC content to retrieve data
* @param tag three-character MARC tag of the data field (must not be
* a control field tag, i.e. must not start with "00")
* @param ind1 first indicator
* @param ind2 second indicator
* @param subfield subfield code whose data value should be returned
* @return {@link Optional} containing the data of the first matching
* subfield, or an empty {@link Optional} if no matching field or
* subfield is found
* @throws IllegalArgumentException if {@code tag} identifies a control
* instead of a data field
*/
public static Optional<String> getValueFromDataField(Record srcRecord, String tag, char ind1, char ind2, char subfield) {
if (Verifier.isControlField(tag)) {
String msg = INVALID_DATA_FIELD_MSG.formatted(tag);
LOGGER.warn("getValueFromDataField:: {}", msg);
throw new IllegalArgumentException(msg);
}

return Optional.ofNullable(computeMarcRecord(srcRecord))
.stream()
.flatMap(marcRecord -> marcRecord.getDataFields().stream())
.filter(df -> df.getTag().equals(tag) && df.getIndicator1() == ind1 && df.getIndicator2() == ind2)
.findFirst()
.flatMap(df -> df.getSubfields(subfield).stream().findFirst().map(Subfield::getData));
}

private static MarcReader buildMarcReader(Record srcRecord) {
Expand All @@ -366,7 +381,7 @@ private static VariableField getSingleFieldByIndicators(List<VariableField> list
return null;
}
return list.stream()
.filter(f -> ((DataField) f).getIndicator1() == INDICATOR && ((DataField) f).getIndicator2() == INDICATOR)
.filter(f -> ((DataField) f).getIndicator1() == INDICATOR_F && ((DataField) f).getIndicator2() == INDICATOR_F)
.findFirst()
.orElse(null);
}
Expand All @@ -387,7 +402,7 @@ private static boolean isField005NeedToUpdate(Record srcRecord, MappingParameter
org.marc4j.marc.Record marcRecord = reader.next();
List<VariableField> variableFields = marcRecord.getVariableFields(TAG_005);
if(!variableFields.isEmpty()) {
VariableField field = variableFields.get(0);
VariableField field = variableFields.getFirst();
needToUpdate = isNotProtected(fieldProtectionSettings, (ControlField) field);
}
}
Expand Down Expand Up @@ -519,8 +534,8 @@ private static boolean removeFirstFoundFieldByName(org.marc4j.marc.Record marcRe
*/
private static boolean isFieldContainsValue(VariableField field, char subfield, String value) {
boolean isContains = false;
if (field instanceof DataField) {
for (Subfield sub : ((DataField) field).getSubfields(subfield)) {
if (field instanceof DataField dataField) {
for (Subfield sub : dataField.getSubfields(subfield)) {
if (isNotEmpty(sub.getData()) && sub.getData().contains(value.trim())) {
isContains = true;
break;
Expand Down
3 changes: 2 additions & 1 deletion src/test/java/org/folio/inventory/dataimport/handlers/MarcBibUpdateKafkaHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import static org.folio.inventory.EntityLinksKafkaTopic.LINKS_STATS;
import static org.folio.inventory.KafkaUtility.checkKafkaEventSent;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.INDICATOR_F;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.SUBFIELD_I;
import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_999;
import static org.folio.inventory.dataimport.util.MappingConstants.MARC_BIB_RECORD_TYPE;
Expand Down Expand Up @@ -258,7 +259,7 @@ public void shouldNotSendSuccessLinkReportEvent(TestContext context) {
public void shouldSendFailedLinkReportEvent(TestContext context) {
// given
var async = context.async();
var instanceId = AdditionalFieldsUtil.getValue(record, TAG_999, SUBFIELD_I)
var instanceId = AdditionalFieldsUtil.getValueFromDataField(record, TAG_999, INDICATOR_F, INDICATOR_F, SUBFIELD_I)
.orElse(null);

record.setId(INVALID_INSTANCE_ID);
Expand Down
7 changes: 7 additions & 0 deletions .../java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ public class CreateInstanceEventHandlerTest {
private static final String PARSED_CONTENT_999ffi = "{\"leader\":\"01314nam 22003851a 4500\",\"fields\":[{\"001\":\"ybp7406411\"},{\"245\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"titleValue\"}]}},{\"336\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"b\":\"b6698d38-149f-11ec-82a8-0242ac130003\"}]}},{\"780\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"Houston oil directory\"}]}},{\"785\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"SAIS review of international affairs\"},{\"x\":\"1945-4724\"}]}},{\"500\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Adaptation of Xi xiang ji by Wang Shifu.\"}]}},{\"520\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\"}]}}]}";
private static final String PARSED_CONTENT_WITH_005 = "{\"leader\":\"01314nam 22003851a 4500\",\"fields\":[{\"001\":\"ybp7406411\"},{\"005\":\"20141107001016.0\"},{\"245\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"titleValue\"}]}},{\"336\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"b\":\"b6698d38-149f-11ec-82a8-0242ac130003\"}]}},{\"780\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"Houston oil directory\"}]}},{\"785\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"SAIS review of international affairs\"},{\"x\":\"1945-4724\"}]}},{\"500\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Adaptation of Xi xiang ji by Wang Shifu.\"}]}},{\"520\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\"}]}}]}";
private static final String PARSED_CONTENT_WITH_999fi = "{\"leader\":\"01314nam 22003851a 4500\",\"fields\":[{\"001\":\"ybp7406411\"},{\"003\":\"in001\"},{\"245\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"titleValue\"}]}},{\"336\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"b\":\"b6698d38-149f-11ec-82a8-0242ac130003\"}]}},{\"780\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"Houston oil directory\"}]}},{\"785\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"SAIS review of international affairs\"},{\"x\":\"1945-4724\"}]}},{\"500\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Adaptation of Xi xiang ji by Wang Shifu.\"}]}},{\"520\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\"}]}} , {\"999\": {\"ind1\":\"f\", \"ind2\":\"f\", \"subfields\":[ { \"i\": \"957985c6-97e3-4038-b0e7-343ecd0b8120\"} ] } }]}";
private static final String PARSED_CONTENT_WITH_999_NON_FF_I = "{\"leader\":\"01314nam 22003851a 4500\",\"fields\":[{\"001\":\"ybp7406411\"},{\"003\":\"in001\"},{\"245\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"titleValue\"}]}},{\"336\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"b\":\"b6698d38-149f-11ec-82a8-0242ac130003\"}]}},{\"780\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"Houston oil directory\"}]}},{\"785\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"SAIS review of international affairs\"},{\"x\":\"1945-4724\"}]}},{\"500\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Adaptation of Xi xiang ji by Wang Shifu.\"}]}},{\"520\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\"}]}} , {\"999\": {\"ind1\":\" \", \"ind2\":\" \", \"subfields\":[ { \"i\": \"957985c6-97e3-4038-b0e7-343ecd0b8120\"} ] } }]}";
private static final String PARSED_CONTENT_WITH_DELETED_05 = "{\"leader\":\"01314dam 22003851a 4500\",\"fields\":[{\"001\":\"ybp7406411\"},{\"003\":\"in001\"},{\"245\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"a\":\"titleValue\"}]}},{\"336\":{\"ind1\":\"1\",\"ind2\":\"0\",\"subfields\":[{\"b\":\"b6698d38-149f-11ec-82a8-0242ac130003\"}]}},{\"780\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"Houston oil directory\"}]}},{\"785\":{\"ind1\":\"0\",\"ind2\":\"0\",\"subfields\":[{\"t\":\"SAIS review of international affairs\"},{\"x\":\"1945-4724\"}]}},{\"500\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Adaptation of Xi xiang ji by Wang Shifu.\"}]}},{\"520\":{\"ind1\":\" \",\"ind2\":\" \",\"subfields\":[{\"a\":\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\"}]}}]}";
private static final String MAPPING_RULES_PATH = "src/test/resources/handlers/bib-rules.json";
private static final String MAPPING_METADATA_URL = "/mapping-metadata";
Expand Down Expand Up @@ -378,6 +379,12 @@ public void shouldProcessEventWith999AndAcceptedInstanceId() throws InterruptedE
shouldProcessEvent(PARSED_CONTENT_WITH_999fi, Boolean.TRUE.toString());
}

@Test
public void shouldProcessEventAndCreateInstanceIfRecordContains999WithSubfieldIWithNonFfIndicators()
throws InterruptedException, ExecutionException, TimeoutException {
shouldProcessEvent(PARSED_CONTENT_WITH_999_NON_FF_I, Boolean.FALSE.toString());
}

public void shouldProcessEvent(String content, String acceptInstanceId) throws InterruptedException, ExecutionException, TimeoutException {
Reader fakeReader = Mockito.mock(Reader.class);

Expand Down
Loading
Loading