436 processing changes

src/main/java/de/bonndan/nivio/input/AppearanceProcessor.java

@@ -25,12 +25,15 @@ public AppearanceProcessor(ProcessLog processLog, IconService iconService) {
         this.iconService = iconService;
     }
 
-    public void process(LandscapeDescription input, Landscape landscape) {
+    @Override
+    public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {
 
         Optional<String> logo = Optional.ofNullable(landscape.getConfig().getBranding().getMapLogo());
         logo.ifPresent(s -> setLandscapeLogo(landscape, s));
 
         landscape.getGroupItems().forEach(group -> group.getItems().forEach(item -> setItemAppearance(group, item)));
+
+        return new ProcessingChangelog();
     }
 
     private void setItemAppearance(Group group, Item item) {

src/main/java/de/bonndan/nivio/input/DiffProcessor.java

@@ -24,16 +24,17 @@ protected DiffProcessor(ProcessLog processLog) {
     }
 
     @Override
-    public void process(LandscapeDescription input, Landscape landscape) {
+    public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {
         Set<Item> existingItems = landscape.getItems().all();
-
+        ProcessingChangelog changelog = new ProcessingChangelog();
         //insert new ones
         List<ItemDescription> newItems = added(input.getItemDescriptions().all(), existingItems, landscape);
         Set<Item> inLandscape = new HashSet<>();
         processLog.info(String.format("Adding %d items in env %s", newItems.size(), landscape.getIdentifier()));
         newItems.forEach(
                 newItem -> {
                     processLog.info(String.format("Creating new item %s in env %s", newItem.getIdentifier(), input.getIdentifier()));
+                    changelog.addEntry(newItem, ProcessingChangelog.ChangeType.CREATED);
                     inLandscape.add(ItemFactory.fromDescription(newItem, landscape));
                 }
         );
@@ -59,9 +60,14 @@ public void process(LandscapeDescription input, Landscape landscape) {
                         }
                     }
 
-                    processLog.info("Updating item " + item.getIdentifier() + " in landscape " + input.getIdentifier());
+                    processLog.info(String.format("Updating item %s in landscape %s", item.getIdentifier(), input.getIdentifier()));
+                    Item newWithAssignedValues = ItemFactory.assignAll(item, description);
+                    inLandscape.add(newWithAssignedValues);
 
-                    inLandscape.add(ItemFactory.assignAll(item, description));
+                    List<String> changes = item.getChanges(newWithAssignedValues);
+                    if (!changes.isEmpty()) {
+                        changelog.addEntry(newWithAssignedValues, ProcessingChangelog.ChangeType.UPDATED, String.join("; ", changes));
+                    }
                 }
         );
 
@@ -72,13 +78,16 @@ public void process(LandscapeDescription input, Landscape landscape) {
         List<Item> toDelete = getUnreferenced(input, inLandscape, existingItems, processLog);
         toDelete.forEach(item -> {
             processLog.info(String.format("Removing item %s from landscape", item));
+            changelog.addEntry(item, ProcessingChangelog.ChangeType.DELETED);
             landscape.getGroup(item.getGroup()).ifPresent(group -> {
                 boolean removed = group.removeItem(item);
                 if (!removed) {
                     LOGGER.warn("Failed to remove item {}", item);
                 }
             });
         });
+
+        return changelog;
     }
 
     private List<Item> getUnreferenced(
@@ -92,7 +101,9 @@ private List<Item> getUnreferenced(
             return new ArrayList<>();
         }
 
-        return removed(kept, all);
+        List<Item> removed = removed(kept, all);
+        logger.info("Removing " + removed.size() + " sources in env " + landscapeDescription.getIdentifier());
+        return removed;
     }
 
     /**
@@ -116,6 +127,8 @@ static List<Item> removed(Collection<Item> items, Collection<Item> itemDescripti
     /**
      * Returns all elements which are not in the second list
      *
+     * @return
+     *
      */
     static List<ItemDescription> added(Collection<ItemDescription> itemDescriptions, Collection<Item> existingItems, Landscape landscape) {
         return itemDescriptions.stream()

src/main/java/de/bonndan/nivio/input/GroupProcessor.java

@@ -25,16 +25,26 @@ protected GroupProcessor(ProcessLog processLog) {
         super(processLog);
     }
 
-    public void process(LandscapeDescription input, Landscape landscape) {
+    public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {
 
+        ProcessingChangelog changelog = new ProcessingChangelog();
         List<Function<String, Boolean>> specs = getSpecs(input.getConfig().getGroupBlacklist());
 
         input.getGroups().forEach((identifier, groupDescription) -> {
             Group g = GroupFactory.createFromDescription(identifier, landscape.getIdentifier(), groupDescription);
 
             if (!isBlacklisted(g.getIdentifier(), specs)) {
-                processLog.info("Adding or updating group " + g.getIdentifier());
-                landscape.addGroup(g);
+
+                Optional<Group> existing = landscape.getGroup(g.getIdentifier());
+                Group added = landscape.addGroup(g);
+                if (existing.isEmpty()) {
+                    processLog.info("Adding group " + g.getIdentifier());
+                    changelog.addEntry(added, ProcessingChangelog.ChangeType.CREATED);
+                } else {
+                    processLog.info("Updating group " + g.getIdentifier());
+                    String updates = String.join("; ", existing.get().getChanges(added));
+                    changelog.addEntry(added, ProcessingChangelog.ChangeType.UPDATED, updates);
+                }
             } else {
                 processLog.info("Ignoring blacklisted group " + g.getIdentifier());
             }
@@ -50,7 +60,9 @@ public void process(LandscapeDescription input, Landscape landscape) {
 
             if (!isBlacklisted(group, specs)) {
                 if (!landscape.getGroups().containsKey(group)) {
-                    landscape.addGroup(GroupFactory.createFromDescription(group, landscape.getIdentifier(), null));
+                    Group fromDescription = GroupFactory.createFromDescription(group, landscape.getIdentifier(), null);
+                    changelog.addEntry(fromDescription, ProcessingChangelog.ChangeType.CREATED, String.format("Reference by item %s", item));
+                    landscape.addGroup(fromDescription);
                 }
             } else {
                 processLog.info("Removing item " + item.getIdentifier() + " because in blacklisted group " + group);
@@ -72,6 +84,8 @@ public void process(LandscapeDescription input, Landscape landscape) {
             }
             throw new RuntimeException(String.format("item group '%s' not found.", item.getGroup()));
         });
+
+        return changelog;
     }
 
     private List<Function<String, Boolean>> getSpecs(List<String> blacklist) {

src/main/java/de/bonndan/nivio/input/Indexer.java

@@ -51,23 +51,26 @@ public void index(final LandscapeDescription input) {
                 });
 
         try {
-            runResolvers(input, landscape);
+            ProcessingChangelog processingChangelog = runResolvers(input, landscape);
             landscapeRepo.save(landscape);
+            eventPublisher.publishEvent(new ProcessingFinishedEvent(input, landscape, processingChangelog));
+            landscape.getLog().info("Reindexed landscape " + input.getIdentifier());
+
         } catch (ProcessingException e) {
             final String msg = "Error while reindexing landscape " + input.getIdentifier();
             landscape.getLog().warn(msg, e);
             eventPublisher.publishEvent(new ProcessingErrorEvent(input.getFullyQualifiedIdentifier(), e));
-            return;
         }
-
-        eventPublisher.publishEvent(new ProcessingFinishedEvent(input, landscape));
-        landscape.getLog().info("Reindexed landscape " + input.getIdentifier());
     }
 
-    private void runResolvers(LandscapeDescription input, Landscape landscape) {
+    private ProcessingChangelog runResolvers(LandscapeDescription input, Landscape landscape) {
 
+        //a detailed textual log
         ProcessLog logger = landscape.getLog();
 
+        //a structured log on component level
+        ProcessingChangelog changelog = new ProcessingChangelog();
+
         // read all input sources
         new SourceReferencesResolver(formatFactory, logger, eventPublisher).resolve(input);
 
@@ -97,19 +100,21 @@ private void runResolvers(LandscapeDescription input, Landscape landscape) {
         new GroupQueryResolver(logger).resolve(input);
 
         // compare landscape against input, add and remove items
-        new DiffProcessor(logger).process(input, landscape);
+        changelog.merge(new DiffProcessor(logger).process(input, landscape));
 
         // assign items to groups, add missing groups
-        new GroupProcessor(logger).process(input, landscape);
+        changelog.merge(new GroupProcessor(logger).process(input, landscape));
 
         // create relations between items
-        new ItemRelationProcessor(logger).process(input, landscape);
+        changelog.merge(new ItemRelationProcessor(logger).process(input, landscape));
 
         // ensures that item have a resolved icon in the api
         new AppearanceProcessor(logger, iconService).process(input, landscape);
 
         // this step must be final or very late to include all item modifications
         landscape.getItems().indexForSearch();
+
+        return changelog;
     }
 
 }

src/main/java/de/bonndan/nivio/input/ItemRelationProcessor.java

@@ -4,9 +4,9 @@
 import de.bonndan.nivio.input.dto.RelationDescription;
 import de.bonndan.nivio.model.*;
 import de.bonndan.nivio.search.ItemMatcher;
+import org.apache.commons.collections.CollectionUtils;
 
-import java.util.Iterator;
-import java.util.Optional;
+import java.util.*;
 
 /**
  * Creates {@link Relation}s between {@link Item}s.
@@ -18,69 +18,134 @@ protected ItemRelationProcessor(ProcessLog processLog) {
     }
 
     @Override
-    public void process(LandscapeDescription input, Landscape landscape) {
-        input.getItemDescriptions().all().forEach(serviceDescription -> {
-            Item origin = landscape.getItems().pick(serviceDescription);
-            if (!input.isPartial()) {
-                processLog.debug(String.format("Clearing relations of %s", origin));
-                origin.getRelations().clear(); //delete all relations on full update
-            }
-        });
+    public ProcessingChangelog process(LandscapeDescription input, Landscape landscape) {
+
+        ProcessingChangelog changelog = new ProcessingChangelog();
+        List<Relation> processed = new ArrayList<>();
 
         input.getItemDescriptions().all().forEach(itemDescription -> {
             Item origin = landscape.getItems().pick(itemDescription);
+            List<Relation> affected = new ArrayList<>();
 
             for (RelationDescription relationDescription : itemDescription.getRelations()) {
-                Optional<Relation> update = update(relationDescription, landscape, origin);
-                update.ifPresent(relation -> {
-                    origin.getRelations().remove(relation);
-                    origin.getRelations().add(relation);
-                    if (relation.getSource() == origin) {
-                        relation.getTarget().getRelations().remove(relation);
-                        relation.getTarget().getRelations().add(relation);
-                    } else {
-                        relation.getSource().getRelations().remove(relation);
-                        relation.getSource().getRelations().add(relation);
-                    }
-                });
+
+                if (!isValid(relationDescription, landscape)) {
+                    continue;
+                }
+
+                Optional<Relation> current = getCurrentRelation(relationDescription, landscape, origin);
+                current.ifPresentOrElse(
+                        (relation) -> updateRelation(changelog, processed, affected, relationDescription, relation),
+                        () -> createRelation(landscape, changelog, processed, origin, affected, relationDescription));
             }
+
+            affected.forEach(relation -> assignToBothEnds(origin, relation));
+            Collection<Relation> toDelete = CollectionUtils.subtract(origin.getRelations(), affected);
+            toDelete.stream()
+                    .filter(relation -> !processed.contains(relation))
+                    .filter(relation -> origin.equals(relation.getSource()))
+                    .filter(relation -> !input.isPartial())
+                    .forEach(relation -> {
+                        removeFromBothEnds(origin, relation);
+                        processLog.info(String.format("Removing relation between %s and %s", relation.getSource(), relation.getTarget()));
+                        changelog.addEntry(relation, ProcessingChangelog.ChangeType.DELETED, null);
+                    });
         });
+
+        return changelog;
     }
 
-    private Optional<Relation> update(RelationDescription relationDescription, Landscape landscape, Item origin) {
+    private void updateRelation(ProcessingChangelog changelog, List<Relation> processed, List<Relation> affected, RelationDescription relationDescription, Relation relation) {
+        Relation update = update(relationDescription, relation);
+        affected.add(update);
+        processed.add(update);
+        processLog.info(String.format("Updating relation between %s and %s", update.getSource(), update.getTarget()));
+        List<String> changes = relation.getChanges(update);
+        if (!changes.isEmpty()) {
+            changelog.addEntry(update, ProcessingChangelog.ChangeType.UPDATED, String.join(";", changes));
+        }
+    }
+
+    private void createRelation(Landscape landscape, ProcessingChangelog changelog, List<Relation> processed, Item origin, List<Relation> affected, RelationDescription relationDescription) {
+        Relation created = create(relationDescription, landscape);
+        affected.add(created);
+        processed.add(created);
+        processLog.info(String.format(origin + ": Adding relation between %s and %s", created.getSource(), created.getTarget()));
+        changelog.addEntry(created, ProcessingChangelog.ChangeType.CREATED, null);
+    }
+
+    private boolean isValid(RelationDescription relationDescription, Landscape landscape) {
 
         Optional<Item> source = findBy(relationDescription.getSource(), landscape);
         if (source.isEmpty()) {
             processLog.warn(String.format("Relation source %s not found", relationDescription.getSource()));
-            return Optional.empty();
+            return false;
         }
 
         Optional<Item> target = findBy(relationDescription.getTarget(), landscape);
         if (target.isEmpty()) {
             processLog.warn(String.format("Relation target %s not found", relationDescription.getTarget()));
-            return Optional.empty();
+            return false;
         }
 
+        return true;
+    }
+
+    private void assignToBothEnds(Item origin, Relation relation) {
+        removeFromBothEnds(origin, relation);
+
+        origin.getRelations().add(relation);
+        if (relation.getSource() == origin) {
+            relation.getTarget().getRelations().add(relation);
+        } else {
+            relation.getSource().getRelations().add(relation);
+        }
+    }
+
+    private void removeFromBothEnds(Item origin, Relation relation) {
+        origin.getRelations().remove(relation);
+        if (relation.getSource() == origin) {
+            relation.getTarget().getRelations().remove(relation);
+        } else {
+            relation.getSource().getRelations().remove(relation);
+        }
+    }
+
+
+    private Optional<Relation> getCurrentRelation(RelationDescription relationDescription,
+                                                  Landscape landscape,
+                                                  Item origin
+    ) {
+        Item source = findBy(relationDescription.getSource(), landscape).orElseThrow();
+        Item target = findBy(relationDescription.getTarget(), landscape).orElseThrow();
+
         Iterator<Relation> iterator = origin.getRelations().iterator();
-        Relation existing = null;
-        Relation created = new Relation(source.get(), target.get());
+        Relation created = new Relation(source, target);
+        Relation existing;
         while (iterator.hasNext()) {
             existing = iterator.next();
             if (existing.equals(created)) {
-                processLog.info(String.format("Updating relation between %s and %s", existing.getSource(), existing.getTarget()));
-                return Optional.of(RelationBuilder.update(existing, relationDescription));
+                return Optional.of(existing);
             }
         }
 
-        created = new Relation(created.getSource(),
-                created.getTarget(),
+        return Optional.empty();
+    }
+
+    private Relation update(RelationDescription relationDescription, Relation existing
+    ) {
+        return RelationBuilder.update(existing, relationDescription);
+    }
+
+    private Relation create(RelationDescription relationDescription, Landscape landscape) {
+
+        return new Relation(
+                findBy(relationDescription.getSource(), landscape).orElseThrow(),
+                findBy(relationDescription.getTarget(), landscape).orElseThrow(),
                 relationDescription.getDescription(),
                 relationDescription.getFormat(),
                 relationDescription.getType()
         );
-
-        processLog.info(String.format("Adding relation from %s to %s", created.getSource(), created.getTarget()));
-        return Optional.of(created);
     }
 
     private Optional<Item> findBy(String term, Landscape landscape) {