Skip to content
This repository has been archived by the owner on Nov 3, 2022. It is now read-only.

Commit

Permalink
Merge pull request #482 from dedica-team/436_processing_changes
Browse files Browse the repository at this point in the history 
436 processing changes
  • Loading branch information
@bonndan
bonndan authored Mar 28, 2021
2 parents 218acda + 6dd1bd5 commit 7f49e77
Show file tree
Hide file tree
Showing 28 changed files with 1,109 additions and 90 deletions.
5 changes: 4 additions & 1 deletion src/main/java/de/bonndan/nivio/input/AppearanceProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,15 @@ public AppearanceProcessor(ProcessLog processLog, IconService iconService) {
this.iconService = iconService;
}

public void process(LandscapeDescription input, Landscape landscape) {
@Override
public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {

Optional<String> logo = Optional.ofNullable(landscape.getConfig().getBranding().getMapLogo());
logo.ifPresent(s -> setLandscapeLogo(landscape, s));

landscape.getGroupItems().forEach(group -> group.getItems().forEach(item -> setItemAppearance(group, item)));

return new ProcessingChangelog();
}

private void setItemAppearance(Group group, Item item) {
Expand Down
23 changes: 18 additions & 5 deletions src/main/java/de/bonndan/nivio/input/DiffProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,16 +24,17 @@ protected DiffProcessor(ProcessLog processLog) {
}

@Override
public void process(LandscapeDescription input, Landscape landscape) {
public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {
Set<Item> existingItems = landscape.getItems().all();

ProcessingChangelog changelog = new ProcessingChangelog();
//insert new ones
List<ItemDescription> newItems = added(input.getItemDescriptions().all(), existingItems, landscape);
Set<Item> inLandscape = new HashSet<>();
processLog.info(String.format("Adding %d items in env %s", newItems.size(), landscape.getIdentifier()));
newItems.forEach(
newItem -> {
processLog.info(String.format("Creating new item %s in env %s", newItem.getIdentifier(), input.getIdentifier()));
changelog.addEntry(newItem, ProcessingChangelog.ChangeType.CREATED);
inLandscape.add(ItemFactory.fromDescription(newItem, landscape));
}
);
Expand All @@ -59,9 +60,14 @@ public void process(LandscapeDescription input, Landscape landscape) {
}
}

processLog.info("Updating item " + item.getIdentifier() + " in landscape " + input.getIdentifier());
processLog.info(String.format("Updating item %s in landscape %s", item.getIdentifier(), input.getIdentifier()));
Item newWithAssignedValues = ItemFactory.assignAll(item, description);
inLandscape.add(newWithAssignedValues);

inLandscape.add(ItemFactory.assignAll(item, description));
List<String> changes = item.getChanges(newWithAssignedValues);
if (!changes.isEmpty()) {
changelog.addEntry(newWithAssignedValues, ProcessingChangelog.ChangeType.UPDATED, String.join("; ", changes));
}
}
);

Expand All @@ -72,13 +78,16 @@ public void process(LandscapeDescription input, Landscape landscape) {
List<Item> toDelete = getUnreferenced(input, inLandscape, existingItems, processLog);
toDelete.forEach(item -> {
processLog.info(String.format("Removing item %s from landscape", item));
changelog.addEntry(item, ProcessingChangelog.ChangeType.DELETED);
landscape.getGroup(item.getGroup()).ifPresent(group -> {
boolean removed = group.removeItem(item);
if (!removed) {
LOGGER.warn("Failed to remove item {}", item);
}
});
});

return changelog;
}

private List<Item> getUnreferenced(
Expand All @@ -92,7 +101,9 @@ private List<Item> getUnreferenced(
return new ArrayList<>();
}

return removed(kept, all);
List<Item> removed = removed(kept, all);
logger.info("Removing " + removed.size() + " sources in env " + landscapeDescription.getIdentifier());
return removed;
}

/**
Expand All @@ -116,6 +127,8 @@ static List<Item> removed(Collection<Item> items, Collection<Item> itemDescripti
/**
* Returns all elements which are not in the second list
*
* @return
*
*/
static List<ItemDescription> added(Collection<ItemDescription> itemDescriptions, Collection<Item> existingItems, Landscape landscape) {
return itemDescriptions.stream()
Expand Down
22 changes: 18 additions & 4 deletions src/main/java/de/bonndan/nivio/input/GroupProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,26 @@ protected GroupProcessor(ProcessLog processLog) {
super(processLog);
}

public void process(LandscapeDescription input, Landscape landscape) {
public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {

ProcessingChangelog changelog = new ProcessingChangelog();
List<Function<String, Boolean>> specs = getSpecs(input.getConfig().getGroupBlacklist());

input.getGroups().forEach((identifier, groupDescription) -> {
Group g = GroupFactory.createFromDescription(identifier, landscape.getIdentifier(), groupDescription);

if (!isBlacklisted(g.getIdentifier(), specs)) {
processLog.info("Adding or updating group " + g.getIdentifier());
landscape.addGroup(g);

Optional<Group> existing = landscape.getGroup(g.getIdentifier());
Group added = landscape.addGroup(g);
if (existing.isEmpty()) {
processLog.info("Adding group " + g.getIdentifier());
changelog.addEntry(added, ProcessingChangelog.ChangeType.CREATED);
} else {
processLog.info("Updating group " + g.getIdentifier());
String updates = String.join("; ", existing.get().getChanges(added));
changelog.addEntry(added, ProcessingChangelog.ChangeType.UPDATED, updates);
}
} else {
processLog.info("Ignoring blacklisted group " + g.getIdentifier());
}
Expand All @@ -50,7 +60,9 @@ public void process(LandscapeDescription input, Landscape landscape) {

if (!isBlacklisted(group, specs)) {
if (!landscape.getGroups().containsKey(group)) {
landscape.addGroup(GroupFactory.createFromDescription(group, landscape.getIdentifier(), null));
Group fromDescription = GroupFactory.createFromDescription(group, landscape.getIdentifier(), null);
changelog.addEntry(fromDescription, ProcessingChangelog.ChangeType.CREATED, String.format("Reference by item %s", item));
landscape.addGroup(fromDescription);
}
} else {
processLog.info("Removing item " + item.getIdentifier() + " because in blacklisted group " + group);
Expand All @@ -72,6 +84,8 @@ public void process(LandscapeDescription input, Landscape landscape) {
}
throw new RuntimeException(String.format("item group '%s' not found.", item.getGroup()));
});

return changelog;
}

private List<Function<String, Boolean>> getSpecs(List<String> blacklist) {
Expand Down
23 changes: 14 additions & 9 deletions src/main/java/de/bonndan/nivio/input/Indexer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,23 +51,26 @@ public void index(final LandscapeDescription input) {
});

try {
runResolvers(input, landscape);
ProcessingChangelog processingChangelog = runResolvers(input, landscape);
landscapeRepo.save(landscape);
eventPublisher.publishEvent(new ProcessingFinishedEvent(input, landscape, processingChangelog));
landscape.getLog().info("Reindexed landscape " + input.getIdentifier());

} catch (ProcessingException e) {
final String msg = "Error while reindexing landscape " + input.getIdentifier();
landscape.getLog().warn(msg, e);
eventPublisher.publishEvent(new ProcessingErrorEvent(input.getFullyQualifiedIdentifier(), e));
return;
}

eventPublisher.publishEvent(new ProcessingFinishedEvent(input, landscape));
landscape.getLog().info("Reindexed landscape " + input.getIdentifier());
}

private void runResolvers(LandscapeDescription input, Landscape landscape) {
private ProcessingChangelog runResolvers(LandscapeDescription input, Landscape landscape) {

//a detailed textual log
ProcessLog logger = landscape.getLog();

//a structured log on component level
ProcessingChangelog changelog = new ProcessingChangelog();

// read all input sources
new SourceReferencesResolver(formatFactory, logger, eventPublisher).resolve(input);

Expand Down Expand Up @@ -97,19 +100,21 @@ private void runResolvers(LandscapeDescription input, Landscape landscape) {
new GroupQueryResolver(logger).resolve(input);

// compare landscape against input, add and remove items
new DiffProcessor(logger).process(input, landscape);
changelog.merge(new DiffProcessor(logger).process(input, landscape));

// assign items to groups, add missing groups
new GroupProcessor(logger).process(input, landscape);
changelog.merge(new GroupProcessor(logger).process(input, landscape));

// create relations between items
new ItemRelationProcessor(logger).process(input, landscape);
changelog.merge(new ItemRelationProcessor(logger).process(input, landscape));

// ensures that item have a resolved icon in the api
new AppearanceProcessor(logger, iconService).process(input, landscape);

// this step must be final or very late to include all item modifications
landscape.getItems().indexForSearch();

return changelog;
}

}
133 changes: 99 additions & 34 deletions src/main/java/de/bonndan/nivio/input/ItemRelationProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
import de.bonndan.nivio.input.dto.RelationDescription;
import de.bonndan.nivio.model.*;
import de.bonndan.nivio.search.ItemMatcher;
import org.apache.commons.collections.CollectionUtils;

import java.util.Iterator;
import java.util.Optional;
import java.util.*;

/**
* Creates {@link Relation}s between {@link Item}s.
Expand All @@ -18,69 +18,134 @@ protected ItemRelationProcessor(ProcessLog processLog) {
}

@Override
public void process(LandscapeDescription input, Landscape landscape) {
input.getItemDescriptions().all().forEach(serviceDescription -> {
Item origin = landscape.getItems().pick(serviceDescription);
if (!input.isPartial()) {
processLog.debug(String.format("Clearing relations of %s", origin));
origin.getRelations().clear(); //delete all relations on full update
}
});
public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {

ProcessingChangelog changelog = new ProcessingChangelog();
List<Relation> processed = new ArrayList<>();

input.getItemDescriptions().all().forEach(itemDescription -> {
Item origin = landscape.getItems().pick(itemDescription);
List<Relation> affected = new ArrayList<>();

for (RelationDescription relationDescription : itemDescription.getRelations()) {
Optional<Relation> update = update(relationDescription, landscape, origin);
update.ifPresent(relation -> {
origin.getRelations().remove(relation);
origin.getRelations().add(relation);
if (relation.getSource() == origin) {
relation.getTarget().getRelations().remove(relation);
relation.getTarget().getRelations().add(relation);
} else {
relation.getSource().getRelations().remove(relation);
relation.getSource().getRelations().add(relation);
}
});

if (!isValid(relationDescription, landscape)) {
continue;
}

Optional<Relation> current = getCurrentRelation(relationDescription, landscape, origin);
current.ifPresentOrElse(
(relation) -> updateRelation(changelog, processed, affected, relationDescription, relation),
() -> createRelation(landscape, changelog, processed, origin, affected, relationDescription));
}

affected.forEach(relation -> assignToBothEnds(origin, relation));
Collection<Relation> toDelete = CollectionUtils.subtract(origin.getRelations(), affected);
toDelete.stream()
.filter(relation -> !processed.contains(relation))
.filter(relation -> origin.equals(relation.getSource()))
.filter(relation -> !input.isPartial())
.forEach(relation -> {
removeFromBothEnds(origin, relation);
processLog.info(String.format("Removing relation between %s and %s", relation.getSource(), relation.getTarget()));
changelog.addEntry(relation, ProcessingChangelog.ChangeType.DELETED, null);
});
});

return changelog;
}

private Optional<Relation> update(RelationDescription relationDescription, Landscape landscape, Item origin) {
private void updateRelation(ProcessingChangelog changelog, List<Relation> processed, List<Relation> affected, RelationDescription relationDescription, Relation relation) {
Relation update = update(relationDescription, relation);
affected.add(update);
processed.add(update);
processLog.info(String.format("Updating relation between %s and %s", update.getSource(), update.getTarget()));
List<String> changes = relation.getChanges(update);
if (!changes.isEmpty()) {
changelog.addEntry(update, ProcessingChangelog.ChangeType.UPDATED, String.join(";", changes));
}
}

private void createRelation(Landscape landscape, ProcessingChangelog changelog, List<Relation> processed, Item origin, List<Relation> affected, RelationDescription relationDescription) {
Relation created = create(relationDescription, landscape);
affected.add(created);
processed.add(created);
processLog.info(String.format(origin + ": Adding relation between %s and %s", created.getSource(), created.getTarget()));
changelog.addEntry(created, ProcessingChangelog.ChangeType.CREATED, null);
}

private boolean isValid(RelationDescription relationDescription, Landscape landscape) {

Optional<Item> source = findBy(relationDescription.getSource(), landscape);
if (source.isEmpty()) {
processLog.warn(String.format("Relation source %s not found", relationDescription.getSource()));
return Optional.empty();
return false;
}

Optional<Item> target = findBy(relationDescription.getTarget(), landscape);
if (target.isEmpty()) {
processLog.warn(String.format("Relation target %s not found", relationDescription.getTarget()));
return Optional.empty();
return false;
}

return true;
}

private void assignToBothEnds(Item origin, Relation relation) {
removeFromBothEnds(origin, relation);

origin.getRelations().add(relation);
if (relation.getSource() == origin) {
relation.getTarget().getRelations().add(relation);
} else {
relation.getSource().getRelations().add(relation);
}
}

private void removeFromBothEnds(Item origin, Relation relation) {
origin.getRelations().remove(relation);
if (relation.getSource() == origin) {
relation.getTarget().getRelations().remove(relation);
} else {
relation.getSource().getRelations().remove(relation);
}
}


private Optional<Relation> getCurrentRelation(RelationDescription relationDescription,
Landscape landscape,
Item origin
) {
Item source = findBy(relationDescription.getSource(), landscape).orElseThrow();
Item target = findBy(relationDescription.getTarget(), landscape).orElseThrow();

Iterator<Relation> iterator = origin.getRelations().iterator();
Relation existing = null;
Relation created = new Relation(source.get(), target.get());
Relation created = new Relation(source, target);
Relation existing;
while (iterator.hasNext()) {
existing = iterator.next();
if (existing.equals(created)) {
processLog.info(String.format("Updating relation between %s and %s", existing.getSource(), existing.getTarget()));
return Optional.of(RelationBuilder.update(existing, relationDescription));
return Optional.of(existing);
}
}

created = new Relation(created.getSource(),
created.getTarget(),
return Optional.empty();
}

private Relation update(RelationDescription relationDescription, Relation existing
) {
return RelationBuilder.update(existing, relationDescription);
}

private Relation create(RelationDescription relationDescription, Landscape landscape) {

return new Relation(
findBy(relationDescription.getSource(), landscape).orElseThrow(),
findBy(relationDescription.getTarget(), landscape).orElseThrow(),
relationDescription.getDescription(),
relationDescription.getFormat(),
relationDescription.getType()
);

processLog.info(String.format("Adding relation from %s to %s", created.getSource(), created.getTarget()));
return Optional.of(created);
}

private Optional<Item> findBy(String term, Landscape landscape) {
Expand Down
Loading

0 comments on commit 7f49e77

Please sign in to comment.