Skip to content

Commit

Permalink
[DSC-1436] - Extend SolrServiceFileInfoPlugin in order to allow cunfi…
Browse files Browse the repository at this point in the history 
…gurable metadata
  • Loading branch information
@NikitaKr1vonosov @vins01-4science
NikitaKr1vonosov authored and vins01-4science committed Mar 21, 2024
1 parent 4064512 commit f0ed448
Show file tree
Hide file tree
Showing 10 changed files with 285 additions and 149 deletions.
172 changes: 30 additions & 142 deletions dspace-api/src/main/java/org/dspace/discovery/SolrServiceFileInfoPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,28 +8,21 @@
package org.dspace.discovery;

import java.sql.SQLException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.BiFunction;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.apache.commons.lang3.StringUtils;
import org.apache.solr.common.SolrInputDocument;
import org.dspace.content.Bitstream;
import org.dspace.content.BitstreamFormat;
import org.dspace.content.Bundle;
import org.dspace.content.MetadataField;
import org.dspace.content.MetadataFieldName;
import org.dspace.content.MetadataValue;
import org.dspace.core.Context;
import org.dspace.discovery.index.adder.IndexAdder;
import org.dspace.discovery.index.mapper.SolrFieldMetadataMapper;
import org.dspace.discovery.indexobject.IndexableItem;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -56,132 +49,23 @@
*/
public class SolrServiceFileInfoPlugin implements SolrServiceIndexPlugin {

/**
* Class used to map a target metadata into a solr index using {@code SolrInputDocument}
*
* @author Vincenzo Mecca (vins01-4science - vincenzo.mecca at 4science.com)
*
* @param <T>
*/
private static class SolrFieldMetadataMapper<T> {
private final String solrField;
private final BiFunction<SolrInputDocument, String, Consumer<T>> fieldAdder;

public SolrFieldMetadataMapper(
String metadata,
BiFunction<SolrInputDocument, String, Consumer<T>> fieldAdder
) {
super();
this.solrField = metadata;
this.fieldAdder = fieldAdder;
}

public void map(SolrInputDocument document, T value) {
this.fieldAdder.apply(document, this.solrField).accept(value);
}

}

private static final Logger logger = LoggerFactory.getLogger(SolrServiceFileInfoPlugin.class);

private static final DateTimeFormatter dtf = DateTimeFormat.forPattern("yyyy-MM-dd");
private static final String BUNDLE_NAME = "ORIGINAL";
private static final String SOLR_FIELD_NAME_FOR_FILENAMES = "original_bundle_filenames";
private static final String SOLR_FIELD_NAME_FOR_DESCRIPTIONS = "original_bundle_descriptions";
private static final String SOLR_FIELD_NAME_FOR_OAIRE_LICENSE_CONDITION = "original_bundle_oaire_licenseCondition";
private static final String SOLR_FIELD_NAME_FOR_DATACITE_RIGHTS = "original_bundle_datacite_rights";
private static final String SOLR_FIELD_NAME_FOR_DATACITE_AVAILABLE = "original_bundle_datacite_available";
private static final String SOLR_FIELD_NAME_FOR_MIMETYPE = "original_bundle_mime_type";
private static final String SOLR_FIELD_NAME_FOR_CHECKSUM = "original_bundle_checksum";
private static final String SOLR_FIELD_NAME_FOR_SIZEBYTES = "original_bundle_sizebytes";
private static final String SOLR_FIELD_NAME_FOR_SHORT_DESCRIPTION = "original_bundle_short_description";
private static final String SOLR_POSTFIX_FILTER = "_filter";
private static final String SOLR_POSTFIX_KEYWORD = "_keyword";
private static final String BITSTREAM_METADATA_SOLR_PREFIX_KEYWORD = "bitstreams.";
// used for facets and filters of type Date to correctly search them and visualize in facets.
private static final String SOLR_POSTFIX_YEAR = ".year";
private static final MetadataFieldName METADATA_DATACITE_RIGHTS = new MetadataFieldName("datacite", "rights");
private static final MetadataFieldName METADATA_DATACITE_AVAILABLE = new MetadataFieldName("datacite", "available");
private static final MetadataFieldName METADATA_LICENSE_CONDITION =
new MetadataFieldName("oaire", "licenseCondition");

private static final BiFunction<SolrInputDocument, String, Consumer<String>> defaultSolrIndexAdder =
(document, fieldName) -> value -> {
Collection<Object> fieldValues = document.getFieldValues(fieldName);
if (fieldValues == null || !fieldValues.contains(value)) {
addField(document, fieldName, value);
addField(document, fieldName.concat(SOLR_POSTFIX_KEYWORD), value);
addField(document, fieldName.concat(SOLR_POSTFIX_FILTER), value);
}
};
private Map<String, SolrFieldMetadataMapper> mappableMetadata;

private static final BiFunction<SolrInputDocument, String, Consumer<String>> simpleSolrIndexAdder =
(document, fieldName) -> value -> {
Collection<Object> fieldValues = document.getFieldValues(fieldName);
if (fieldValues == null || !fieldValues.contains(value)) {
addField(document, fieldName, value);
}
};
private IndexAdder simpleIndexAdder;

private static final BiFunction<SolrInputDocument, String, Consumer<String>> bitstreamMetadataSolrIndexAdder =
(document, fieldName) -> value -> {
String baseIndex = BITSTREAM_METADATA_SOLR_PREFIX_KEYWORD.concat(fieldName);
Collection<Object> fieldValues = document.getFieldValues(baseIndex);
if (fieldValues == null || !fieldValues.contains(value)) {
addField(document, baseIndex, value);
addField(document, baseIndex.concat(SOLR_POSTFIX_KEYWORD), value);
addField(document, baseIndex.concat(SOLR_POSTFIX_FILTER), value);
}
};
private IndexAdder defaultIndexAdder;

private static final BiFunction<SolrInputDocument, String, Consumer<String>> yearSolrIndexAdder =
(document, fieldName) -> value -> {
Collection<Object> fieldValues = document.getFieldValues(fieldName);
if (fieldValues == null || !fieldValues.contains(value)) {
addField(document, fieldName, value);
addField(document, fieldName.concat(SOLR_POSTFIX_KEYWORD), value);
addField(document, fieldName.concat(SOLR_POSTFIX_FILTER), value);
addField(document, fieldName.concat(SOLR_POSTFIX_YEAR), dtf.parseLocalDate(value).getYear());
}
};

private static final SolrFieldMetadataMapper<String> getFieldMapper(
String solrField,
BiFunction<SolrInputDocument, String, Consumer<String>> adder
) {
return new SolrFieldMetadataMapper<String>(solrField, adder);
}

private static final SolrFieldMetadataMapper<String> OAIRE_LICENSE_MAPPER =
new SolrFieldMetadataMapper<String>(
SOLR_FIELD_NAME_FOR_OAIRE_LICENSE_CONDITION,
defaultSolrIndexAdder
);

private static final SolrFieldMetadataMapper<String> DATACITE_RIGHTS_MAPPER =
new SolrFieldMetadataMapper<String>(
SOLR_FIELD_NAME_FOR_DATACITE_RIGHTS,
defaultSolrIndexAdder
);

private static final SolrFieldMetadataMapper<String> DATACITE_AVAILABLE_MAPPER =
new SolrFieldMetadataMapper<String>(
SOLR_FIELD_NAME_FOR_DATACITE_AVAILABLE,
yearSolrIndexAdder
);

private static final Map<String, SolrFieldMetadataMapper<String>> mappableMetadatas =
Stream.of(
Map.entry(METADATA_LICENSE_CONDITION.toString(), OAIRE_LICENSE_MAPPER),
Map.entry(METADATA_DATACITE_RIGHTS.toString(), DATACITE_RIGHTS_MAPPER),
Map.entry(METADATA_DATACITE_AVAILABLE.toString(), DATACITE_AVAILABLE_MAPPER)
)
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));


private static void addField(SolrInputDocument document, String name, Object value) {
document.addField(name, value);
}
private IndexAdder bitstreamMetadataIndexAdder;

@Override
public void additionalIndex(Context context, IndexableObject indexableObject, SolrInputDocument document) {
Expand Down Expand Up @@ -221,15 +105,15 @@ private void generateBitstreamIndex(Context context, SolrInputDocument document,

private void indexBitstreamFields(Context context, SolrInputDocument document, Bitstream bitstream) {
addAndHandleException(
simpleSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_FILENAMES, bitstream.getName()
simpleIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_FILENAMES, bitstream.getName()
);

Optional.ofNullable(bitstream.getDescription())
.filter(StringUtils::isNotEmpty)
.ifPresent(
(description) ->
addAndHandleException(
simpleSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_DESCRIPTIONS, description
simpleIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_DESCRIPTIONS, description
)
);

Expand All @@ -243,15 +127,15 @@ private void indexBitstreamFields(Context context, SolrInputDocument document, B
.filter(StringUtils::isNotBlank)
.ifPresent(format ->
addAndHandleException(
defaultSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_MIMETYPE, format
defaultIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_MIMETYPE, format
)
);

formatOptional
.map(BitstreamFormat::getShortDescription)
.ifPresent(format ->
addAndHandleException(
simpleSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_SHORT_DESCRIPTION, format
defaultIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_SHORT_DESCRIPTION, format
)
);
} catch (SQLException e) {
Expand All @@ -264,7 +148,7 @@ private void indexBitstreamFields(Context context, SolrInputDocument document, B
.map(checksum -> bitstream.getChecksumAlgorithm() + ":" + bitstream.getChecksum())
.ifPresent(checksum ->
addAndHandleException(
defaultSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_CHECKSUM, checksum
defaultIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_CHECKSUM, checksum
)
);

Expand All @@ -273,18 +157,18 @@ private void indexBitstreamFields(Context context, SolrInputDocument document, B
.map(String::valueOf)
.ifPresent(size ->
addAndHandleException(
simpleSolrIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_SIZEBYTES, size
simpleIndexAdder, document, bitstream, SOLR_FIELD_NAME_FOR_SIZEBYTES, size
)
);
}

protected void addAndHandleException(
BiFunction<SolrInputDocument, String, Consumer<String>> solrIndexAdder,
IndexAdder indexAdder,
SolrInputDocument document, Bitstream bitstream,
String field, String value
) {
try {
solrIndexAdder.apply(document, field).accept(value);
indexAdder.add(document, field, value);
} catch (Exception e) {
logger.warn(
"Error occurred during the update of index field {} for bitstream {}",
Expand All @@ -302,21 +186,25 @@ private void indexBitstreamsMetadatadas(SolrInputDocument document, Bitstream bi
.forEach(metadata -> {
MetadataField metadataField = metadata.getMetadataField();
String bitstreamMetadata = metadataField.toString('.');
Optional.ofNullable(mappableMetadatas.get(bitstreamMetadata))
.filter(Objects::nonNull)
.orElse(
getFieldMapper(
metadataField.toString(),
bitstreamMetadataSolrIndexAdder
)
)
Optional.ofNullable(mappableMetadata.get(bitstreamMetadata))
.orElseGet(() -> new SolrFieldMetadataMapper(metadataField.toString(), bitstreamMetadataIndexAdder))
.map(document, metadata.getValue());
});
}

private <T> boolean areEquals(MetadataFieldName metadataFieldName, MetadataValue metadata) {
return StringUtils.equals(metadataFieldName.schema, metadata.getSchema()) &&
StringUtils.equals(metadataFieldName.element, metadata.getElement()) &&
StringUtils.equals(metadataFieldName.qualifier, metadata.getQualifier());
public void setMappableMetadata(Map<String, SolrFieldMetadataMapper> mappableMetadata) {
this.mappableMetadata = mappableMetadata;
}

public void setSimpleIndexAdder(IndexAdder simpleIndexAdder) {
this.simpleIndexAdder = simpleIndexAdder;
}

public void setDefaultIndexAdder(IndexAdder defaultIndexAdder) {
this.defaultIndexAdder = defaultIndexAdder;
}

public void setBitstreamMetadataIndexAdder(IndexAdder bitstreamMetadataIndexAdder) {
this.bitstreamMetadataIndexAdder = bitstreamMetadataIndexAdder;
}
}
31 changes: 31 additions & 0 deletions ...e-api/src/main/java/org/dspace/discovery/index/adder/BitstreamMetadataSolrIndexAdder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.discovery.index.adder;

import java.util.Collection;

import org.apache.solr.common.SolrInputDocument;

/**
* Implementation of {@link IndexAdder} defining default bitstream metadata index adder.
*/
public class BitstreamMetadataSolrIndexAdder implements IndexAdder {

private static final String BITSTREAM_METADATA_SOLR_PREFIX_KEYWORD = "bitstreams.";

@Override
public void add(SolrInputDocument document, String solrFieldName, String value) {
String baseIndex = BITSTREAM_METADATA_SOLR_PREFIX_KEYWORD.concat(solrFieldName);
Collection<Object> fieldValues = document.getFieldValues(baseIndex);
if (fieldValues == null || !fieldValues.contains(value)) {
document.addField(solrFieldName, value);
document.addField(solrFieldName.concat(SOLR_POSTFIX_KEYWORD), value);
document.addField(solrFieldName.concat(SOLR_POSTFIX_FILTER), value);
}
}
}
28 changes: 28 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/index/adder/DefaultSolrIndexAdder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.discovery.index.adder;

import java.util.Collection;

import org.apache.solr.common.SolrInputDocument;

/**
* Implementation of {@link IndexAdder} defining default index adder.
*/
public class DefaultSolrIndexAdder implements IndexAdder {

@Override
public void add(SolrInputDocument document, String solrFieldName, String value) {
Collection<Object> fieldValues = document.getFieldValues(solrFieldName);
if (fieldValues == null || !fieldValues.contains(value)) {
document.addField(solrFieldName, value);
document.addField(solrFieldName.concat(SOLR_POSTFIX_KEYWORD), value);
document.addField(solrFieldName.concat(SOLR_POSTFIX_FILTER), value);
}
}
}
29 changes: 29 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/index/adder/IndexAdder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.discovery.index.adder;

import org.apache.solr.common.SolrInputDocument;

/**
* Interface for defining strategy of adding an index.
*/
public interface IndexAdder {

String SOLR_POSTFIX_FILTER = "_filter";

String SOLR_POSTFIX_KEYWORD = "_keyword";

/**
* Adds new index to the document based on passed solr field name and value.
*
* @param document document which will get a new index
* @param solrField solr field name
* @param value solr field value.
*/
void add(SolrInputDocument document, String solrField, String value);
}
26 changes: 26 additions & 0 deletions dspace-api/src/main/java/org/dspace/discovery/index/adder/SimpleSolrIndexAdder.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
/**
* The contents of this file are subject to the license and copyright
* detailed in the LICENSE and NOTICE files at the root of the source
* tree and available online at
*
* http://www.dspace.org/license/
*/
package org.dspace.discovery.index.adder;

import java.util.Collection;

import org.apache.solr.common.SolrInputDocument;

/**
* Implementation of {@link IndexAdder} defining simple index adder without any postfixes.
*/
public class SimpleSolrIndexAdder implements IndexAdder {

@Override
public void add(SolrInputDocument document, String solrFieldName, String value) {
Collection<Object> fieldValues = document.getFieldValues(solrFieldName);
if (fieldValues == null || !fieldValues.contains(value)) {
document.addField(solrFieldName, value);
}
}
}
Loading

0 comments on commit f0ed448

Please sign in to comment.