diff --git a/nifi-api/src/main/java/org/apache/nifi/asset/Asset.java b/nifi-api/src/main/java/org/apache/nifi/asset/Asset.java new file mode 100644 index 000000000000..9ee144150326 --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/asset/Asset.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import java.io.File; +import java.util.Optional; + +/** + * An Asset is a representation of some resource that is necessary in order to run a dataflow. + * An Asset is always accessed as a local file. + */ +public interface Asset { + + /** + * Returns a unique identifier for the Asset + */ + String getIdentifier(); + + /** + * Returns the identifier of the parameter context the Asset belongs to + */ + String getParameterContextIdentifier(); + + /** + * Returns the name of the Asset + */ + String getName(); + + /** + * Returns the local file that the Asset is associated with + */ + File getFile(); + + /** + * Returns the digest of the contents of the local file that the Asset is associated with. + * The digest will not be present when the asset is considered missing and the local file does not exist. + */ + Optional getDigest(); +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedAsset.java b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedAsset.java new file mode 100644 index 000000000000..2f1acf66d2fc --- /dev/null +++ b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedAsset.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.flow; + +import io.swagger.v3.oas.annotations.media.Schema; + +public class VersionedAsset { + private String identifier; + private String name; + + @Schema(description = "The identifier of the asset") + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(final String identifier) { + this.identifier = identifier; + } + + @Schema(description = "The name of the asset") + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + +} diff --git a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedParameter.java b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedParameter.java index 987e23ad4d3d..4daa3bd4c286 100644 --- a/nifi-api/src/main/java/org/apache/nifi/flow/VersionedParameter.java +++ b/nifi-api/src/main/java/org/apache/nifi/flow/VersionedParameter.java @@ -18,6 +18,7 @@ import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; import java.util.Objects; public class VersionedParameter { @@ -27,6 +28,7 @@ public class VersionedParameter { private boolean sensitive; private boolean provided; private String value; + private List referencedAssets; @Schema(description = "The name of the parameter") public String getName() { @@ -73,6 +75,15 @@ public void setValue(String value) { this.value = value; } + @Schema(description = "The assets that are referenced by this parameter") + public List getReferencedAssets() { + return referencedAssets; + } + + public void setReferencedAssets(final List referencedAssets) { + this.referencedAssets = referencedAssets; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java b/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java index d9ec1810eb7f..ff3030f2ca49 100644 --- a/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java +++ b/nifi-api/src/main/java/org/apache/nifi/parameter/Parameter.java @@ -16,27 +16,39 @@ */ package org.apache.nifi.parameter; +import org.apache.nifi.asset.Asset; + +import java.io.File; +import java.util.List; import java.util.Objects; +import java.util.stream.Collectors; public class Parameter { private final ParameterDescriptor descriptor; private final String value; private final String parameterContextId; private final boolean provided; + private final List referencedAssets; - public Parameter(final ParameterDescriptor descriptor, final String value, final String parameterContextId, final Boolean provided) { - this.descriptor = descriptor; - this.value = value; - this.parameterContextId = parameterContextId; - this.provided = provided == null ? false : provided.booleanValue(); - } + private Parameter(final Builder builder) { + this.descriptor = new ParameterDescriptor.Builder() + .name(builder.name) + .description(builder.description) + .sensitive(builder.sensitive) + .build(); - public Parameter(final Parameter parameter, final String parameterContextId) { - this(parameter.getDescriptor(), parameter.getValue(), parameterContextId, parameter.isProvided()); - } + this.parameterContextId = builder.parameterContextId; + this.provided = builder.provided; - public Parameter(final ParameterDescriptor descriptor, final String value) { - this(descriptor, value, null, false); + this.referencedAssets = builder.referencedAssets; + if (this.referencedAssets == null || this.referencedAssets.isEmpty()) { + this.value = builder.value; + } else { + this.value = referencedAssets.stream() + .map(Asset::getFile) + .map(File::getAbsolutePath) + .collect(Collectors.joining(",")); + } } public ParameterDescriptor getDescriptor() { @@ -47,6 +59,10 @@ public String getValue() { return value; } + public List getReferencedAssets() { + return referencedAssets; + } + public String getParameterContextId() { return parameterContextId; } @@ -62,7 +78,10 @@ public boolean equals(final Object o) { } final Parameter parameter = (Parameter) o; - return Objects.equals(descriptor, parameter.descriptor) && Objects.equals(value, parameter.value); + return Objects.equals(descriptor, parameter.descriptor) + && Objects.equals(value, parameter.value) + && Objects.equals(parameterContextId, parameter.parameterContextId) + && Objects.equals(referencedAssets, parameter.referencedAssets); } @Override @@ -77,4 +96,84 @@ public int hashCode() { public boolean isProvided() { return provided; } + + public static class Builder { + private String name; + private String description; + private boolean sensitive; + private String value; + private String parameterContextId; + private boolean provided; + private List referencedAssets = List.of(); + + public Builder fromParameter(final Parameter parameter) { + descriptor(parameter.getDescriptor()); + this.parameterContextId = parameter.getParameterContextId(); + this.provided = parameter.isProvided(); + this.referencedAssets = parameter.getReferencedAssets() == null ? List.of() : parameter.getReferencedAssets(); + if (this.referencedAssets.isEmpty()) { + this.value = parameter.getValue(); + } + + return this; + } + + public Builder descriptor(final ParameterDescriptor descriptor) { + this.name = descriptor.getName(); + this.description = descriptor.getDescription(); + this.sensitive = descriptor.isSensitive(); + return this; + } + + public Builder name(final String name) { + this.name = name; + return this; + } + + public Builder description(final String description) { + this.description = description; + return this; + } + + public Builder sensitive(final boolean sensitive) { + this.sensitive = sensitive; + return this; + } + + public Builder value(final String value) { + this.value = value; + this.referencedAssets = List.of(); + return this; + } + + public Builder parameterContextId(final String parameterContextId) { + this.parameterContextId = parameterContextId; + return this; + } + + public Builder provided(final Boolean provided) { + this.provided = provided != null && provided; + return this; + } + + public Builder referencedAssets(final List referencedAssets) { + this.referencedAssets = referencedAssets == null ? List.of() : referencedAssets; + if (!this.referencedAssets.isEmpty()) { + this.value = null; + } + + return this; + } + + public Parameter build() { + if (name == null) { + throw new IllegalStateException("Name or Descriptor is required"); + } + if (value != null && referencedAssets != null && !referencedAssets.isEmpty()) { + throw new IllegalStateException("A Parameter's value or referenced assets may be set but not both"); + } + + return new Parameter(this); + } + } } diff --git a/nifi-api/src/test/java/org/apache/nifi/parameter/TestParameter.java b/nifi-api/src/test/java/org/apache/nifi/parameter/TestParameter.java new file mode 100644 index 000000000000..cc366ff57e60 --- /dev/null +++ b/nifi-api/src/test/java/org/apache/nifi/parameter/TestParameter.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.parameter; + +import org.apache.nifi.asset.Asset; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.util.List; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class TestParameter { + + @Test + public void testCreateParameterWithValue() { + final Parameter parameter = new Parameter.Builder() + .name("A") + .value("value") + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals("value", parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertTrue(parameter.getReferencedAssets().isEmpty()); + } + + @Test + public void testCreateParameterWithSingleReference() { + final File file = new File("file"); + final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest"); + + final Parameter parameter = new Parameter.Builder() + .name("A") + .referencedAssets(List.of(asset)) + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals(file.getAbsolutePath(), parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(1, parameter.getReferencedAssets().size()); + assertEquals(asset, parameter.getReferencedAssets().getFirst()); + } + + @Test + public void testCreateParameterWithMultipleReferences() { + final File file1 = new File("file1"); + final File file2 = new File("file2"); + final File file3 = new File("file3"); + final Asset asset1 = new MockAsset("id1", "parmContext", "name1", file1, "asset-digest"); + final Asset asset2 = new MockAsset("id2", "parmContext", "name2", file2, "asset-digest"); + final Asset asset3 = new MockAsset("id3", "parmContext", "name3", file3, "asset-digest"); + + final Parameter parameter = new Parameter.Builder() + .name("A") + .referencedAssets(List.of(asset1, asset2, asset3)) + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals(file1.getAbsolutePath() + "," + file2.getAbsolutePath() + "," + file3.getAbsolutePath(), parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(3, parameter.getReferencedAssets().size()); + } + + @Test + public void testCreateParameterWithValueThenAsset() { + final File file = new File("file"); + final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest"); + + final Parameter parameter = new Parameter.Builder() + .name("A") + .value("value") + .referencedAssets(List.of(asset)) + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals(file.getAbsolutePath(), parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(1, parameter.getReferencedAssets().size()); + assertEquals(asset, parameter.getReferencedAssets().getFirst()); + } + + @Test + public void testCreateParameterAssetThenValue() { + final File file = new File("file"); + final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest"); + + final Parameter parameter = new Parameter.Builder() + .name("A") + .referencedAssets(List.of(asset)) + .value("value") + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals("value", parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(0, parameter.getReferencedAssets().size()); + } + + @Test + public void testCreateParameterFromOtherThenOverrideWithAsset() { + final File file = new File("file"); + final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest"); + + final Parameter original = new Parameter.Builder() + .name("A") + .value("value") + .build(); + + final Parameter parameter = new Parameter.Builder() + .fromParameter(original) + .referencedAssets(List.of(asset)) + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(1, parameter.getReferencedAssets().size()); + assertEquals(asset, parameter.getReferencedAssets().getFirst()); + } + + @Test + public void testCreateParameterFromOtherThenOverrideWithValue() { + final File file = new File("file"); + final Asset asset = new MockAsset("id", "parmContext", "name", file, "asset-digest"); + + final Parameter original = new Parameter.Builder() + .name("A") + .referencedAssets(List.of(asset)) + .build(); + + final Parameter parameter = new Parameter.Builder() + .fromParameter(original) + .value("value") + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals("value", parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(0, parameter.getReferencedAssets().size()); + } + + @Test + public void testCreateParameterFromOtherThenOverrideWithDifferentValue() { + final Parameter original = new Parameter.Builder() + .name("A") + .value("value 1") + .build(); + + final Parameter parameter = new Parameter.Builder() + .fromParameter(original) + .value("value 2") + .build(); + + assertEquals("A", parameter.getDescriptor().getName()); + assertEquals("value 2", parameter.getValue()); + assertNotNull(parameter.getReferencedAssets()); + assertEquals(0, parameter.getReferencedAssets().size()); + } + + private static class MockAsset implements Asset { + private final String identifier; + private final String parameterContextIdentifier; + private final String name; + private final File file; + private final String digest; + + public MockAsset(final String identifier, final String parameterContextIdentifier, final String name, final File file, final String digest) { + this.identifier = identifier; + this.parameterContextIdentifier = parameterContextIdentifier; + this.name = name; + this.file = file; + this.digest = digest; + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public String getParameterContextIdentifier() { + return parameterContextIdentifier; + } + + @Override + public String getName() { + return name; + } + + @Override + public File getFile() { + return file; + } + + @Override + public Optional getDigest() { + return Optional.ofNullable(digest); + } + } +} diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java index 403a2ee7a996..07115dab63ec 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestQuery.java @@ -209,7 +209,7 @@ public void testEvaluateELStringSensitiveParameters() { final ParameterLookup parameterLookup = mock(ParameterLookup.class); final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).sensitive(true).build(); - final Parameter parameter = new Parameter(parameterDescriptor, value); + final Parameter parameter = new Parameter.Builder().descriptor(parameterDescriptor).value(value).build(); when(parameterLookup.getParameter(eq(parameterName))).thenReturn(Optional.of(parameter)); when(parameterLookup.isEmpty()).thenReturn(false); @@ -2619,7 +2619,7 @@ public Optional getParameter(final String parameterName) { return Optional.empty(); } - return Optional.of(new Parameter(new ParameterDescriptor.Builder().name(parameterName).build(), value)); + return Optional.of(new Parameter.Builder().name(parameterName).value(value).build()); } @Override diff --git a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java index 3023bc515f5a..607e3bd4b338 100644 --- a/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java +++ b/nifi-commons/nifi-expression-language/src/test/java/org/apache/nifi/attribute/expression/language/TestStandardPreparedQuery.java @@ -249,8 +249,8 @@ public void testPreparedQueryWithOr() { @Test public void testSensitiveParameter() { final Map parameters = new HashMap<>(); - parameters.put("param", new Parameter(new ParameterDescriptor.Builder().name("param").build(), "value")); - parameters.put("sensi", new Parameter(new ParameterDescriptor.Builder().name("sensi").sensitive(true).build(), "secret")); + parameters.put("param", new Parameter.Builder().name("param").value("value").build()); + parameters.put("sensi", new Parameter.Builder().name("sensi").sensitive(true).value("secret").build()); final ParameterLookup parameterLookup = new ParameterLookup() { @Override @@ -286,7 +286,7 @@ public void testEvaluateExpressionLanguageVariableValueSensitiveParameterReferen final ParameterLookup parameterLookup = mock(ParameterLookup.class); final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).sensitive(true).build(); - final Parameter parameter = new Parameter(parameterDescriptor, parameterValue); + final Parameter parameter = new Parameter.Builder().descriptor(parameterDescriptor).value(parameterValue).build(); when(parameterLookup.getParameter(eq(parameterName))).thenReturn(Optional.of(parameter)); when(parameterLookup.isEmpty()).thenReturn(false); diff --git a/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestStandardParameterTokenList.java b/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestStandardParameterTokenList.java index 342510ced91d..4d6059e87002 100644 --- a/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestStandardParameterTokenList.java +++ b/nifi-commons/nifi-parameter/src/test/java/org/apache/nifi/parameter/TestStandardParameterTokenList.java @@ -34,8 +34,8 @@ public void testSubstitute() { referenceList.add(new StandardParameterReference("foo", 0, 5, "#{foo}")); final ParameterLookup paramLookup = Mockito.mock(ParameterLookup.class); - Mockito.when(paramLookup.getParameter("foo")).thenReturn(Optional.of(new Parameter(new ParameterDescriptor.Builder().name("foo").build(), "bar"))); - Mockito.when(paramLookup.getParameter("bazz")).thenReturn(Optional.of(new Parameter(new ParameterDescriptor.Builder().name("bazz").build(), "baz"))); + Mockito.when(paramLookup.getParameter("foo")).thenReturn(Optional.of(new Parameter.Builder().name("foo").value("bar").build())); + Mockito.when(paramLookup.getParameter("bazz")).thenReturn(Optional.of(new Parameter.Builder().name("bazz").value("baz").build())); StandardParameterTokenList references = new StandardParameterTokenList("#{foo}", referenceList); assertEquals("bar", references.substitute(paramLookup)); @@ -71,7 +71,7 @@ public void testSubstituteWithEscapes() { referenceList.add(new EscapedParameterReference(2, 8, "##{foo}")); final ParameterLookup paramContext = Mockito.mock(ParameterLookup.class); - Mockito.when(paramContext.getParameter("foo")).thenReturn(Optional.of(new Parameter(new ParameterDescriptor.Builder().name("foo").build(), "bar"))); + Mockito.when(paramContext.getParameter("foo")).thenReturn(Optional.of(new Parameter.Builder().name("foo").value("bar").build())); StandardParameterTokenList references = new StandardParameterTokenList("####{foo}", referenceList); assertEquals("##{foo}", references.substitute(paramContext)); diff --git a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java index 7cfbf3af8190..5dc0b634043b 100644 --- a/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java +++ b/nifi-commons/nifi-properties/src/main/java/org/apache/nifi/util/NiFiProperties.java @@ -125,6 +125,9 @@ public class NiFiProperties extends ApplicationProperties { public static final String PROVENANCE_JOURNAL_COUNT = "nifi.provenance.repository.journal.count"; public static final String PROVENANCE_REPO_DEBUG_FREQUENCY = "nifi.provenance.repository.debug.frequency"; + public static final String ASSET_MANAGER_IMPLEMENTATION = "nifi.asset.manager.implementation"; + public static final String ASSET_MANAGER_PREFIX = "nifi.asset.manager.properties."; + // status repository properties public static final String COMPONENT_STATUS_REPOSITORY_IMPLEMENTATION = "nifi.components.status.repository.implementation"; public static final String COMPONENT_STATUS_SNAPSHOT_FREQUENCY = "nifi.components.status.snapshot.frequency"; diff --git a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java index b40c76a1c914..d3cf56bbe6b7 100644 --- a/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java +++ b/nifi-commons/nifi-utils/src/main/java/org/apache/nifi/util/file/FileUtils.java @@ -582,4 +582,39 @@ public static long getContainerUsableSpace(final Path path) { return path.toFile().getUsableSpace(); } + // The invalid character list is derived from this Stackoverflow page. + // https://stackoverflow.com/questions/1155107/is-there-a-cross-platform-java-method-to-remove-filename-special-chars + private final static int[] INVALID_CHARS = {34, 60, 62, 124, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 58, 42, 63, 92, 47, 32}; + + static { + Arrays.sort(INVALID_CHARS); + } + + /** + * Replaces invalid characters for a file system name within a given filename string to underscore '_'. + * Be careful not to pass a file path as this method replaces path delimiter characters (i.e forward/back slashes). + * @param filename The filename to clean + * @return sanitized filename + */ + public static String getSanitizedFilename(String filename) { + if (filename == null) { + return null; + } + if (filename.isEmpty()) { + return ""; + } + + int codePointCount = filename.codePointCount(0, filename.length()); + final StringBuilder cleanName = new StringBuilder(); + for (int i = 0; i < codePointCount; i++) { + int c = filename.codePointAt(i); + if (Arrays.binarySearch(INVALID_CHARS, c) < 0) { + cleanName.appendCodePoint(c); + } else { + cleanName.append('_'); + } + } + return cleanName.toString(); + } } diff --git a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc index 1c95c8e33589..5aa3721a9d82 100644 --- a/nifi-docs/src/main/asciidoc/toolkit-guide.adoc +++ b/nifi-docs/src/main/asciidoc/toolkit-guide.adoc @@ -146,6 +146,12 @@ The following are available commands: nifi delete-nar nifi list-nars nifi list-nar-component-types + nifi create-asset + nifi list-assets + nifi get-asset + nifi delete-asset + nifi add-asset-reference + nifi remove-asset-reference registry current-user registry list-buckets registry create-bucket diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java index addc212d589f..194bae32f571 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/main/java/org/apache/nifi/parameter/aws/AwsSecretsManagerParameterProvider.java @@ -44,20 +44,19 @@ import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.parameter.AbstractParameterProvider; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.VerifiableParameterProvider; import org.apache.nifi.processor.util.StandardValidators; import org.apache.nifi.processors.aws.credentials.provider.service.AWSCredentialsProviderService; import org.apache.nifi.ssl.SSLContextService; -import javax.net.ssl.SSLContext; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import javax.net.ssl.SSLContext; /** * Reads secrets from AWS Secrets Manager to provide parameter values. Secrets must be created similar to the following AWS cli command:

@@ -232,8 +231,11 @@ private List fetchSecret(final AWSSecretsManager secretsManager, } private Parameter createParameter(final String parameterName, final String parameterValue) { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); - return new Parameter(parameterDescriptor, parameterValue, null, true); + return new Parameter.Builder() + .name(parameterName) + .value(parameterValue) + .provided(true) + .build(); } protected ClientConfiguration createConfiguration(final ConfigurationContext context) { diff --git a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java index 9e65581f6ba2..272f406a14a3 100644 --- a/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java +++ b/nifi-extension-bundles/nifi-aws-bundle/nifi-aws-parameter-providers/src/test/java/org/apache/nifi/parameter/aws/TestAwsSecretsManagerParameterProvider.java @@ -28,7 +28,6 @@ import org.apache.nifi.components.ConfigVerificationResult; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.VerifiableParameterProvider; import org.apache.nifi.reporting.InitializationException; @@ -187,7 +186,10 @@ private List runProviderTest(final AWSSecretsManager secretsMana } private static Parameter parameter(final String name, final String value) { - return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value); + return new Parameter.Builder() + .name(name) + .value(value) + .build(); } private static ArgumentMatcher matchesGetSecretValueRequest(final String groupName) { diff --git a/nifi-extension-bundles/nifi-azure-bundle/nifi-azure-parameter-providers/src/main/java/org/apache/nifi/parameter/azure/AzureKeyVaultSecretsParameterProvider.java b/nifi-extension-bundles/nifi-azure-bundle/nifi-azure-parameter-providers/src/main/java/org/apache/nifi/parameter/azure/AzureKeyVaultSecretsParameterProvider.java index fbd01c49d4f8..78107a76ffc0 100644 --- a/nifi-extension-bundles/nifi-azure-bundle/nifi-azure-parameter-providers/src/main/java/org/apache/nifi/parameter/azure/AzureKeyVaultSecretsParameterProvider.java +++ b/nifi-extension-bundles/nifi-azure-bundle/nifi-azure-parameter-providers/src/main/java/org/apache/nifi/parameter/azure/AzureKeyVaultSecretsParameterProvider.java @@ -28,7 +28,6 @@ import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.parameter.AbstractParameterProvider; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.VerifiableParameterProvider; import org.apache.nifi.processor.util.StandardValidators; @@ -180,8 +179,11 @@ private List createParameterGroupFromMap(final Map runProviderTest(final SecretManagerServiceClient se } private static Parameter parameter(final String name, final String value) { - return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value); + return new Parameter.Builder() + .name(name) + .value(value) + .build(); } } diff --git a/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultParameterProvider.java b/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultParameterProvider.java index a073f07b6f80..624e976aef43 100644 --- a/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultParameterProvider.java +++ b/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/main/java/org/apache/nifi/vault/hashicorp/HashiCorpVaultParameterProvider.java @@ -25,7 +25,6 @@ import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.parameter.AbstractParameterProvider; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.parameter.VerifiableParameterProvider; @@ -104,8 +103,11 @@ private List getParameterGroups(final HashiCorpVaultCommunicatio final Map keyValues = vaultCommunicationService.readKeyValueSecretMap(kvPath, secretName); final List parameters = new ArrayList<>(); keyValues.forEach( (key, value) -> { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(key).build(); - parameters.add(new Parameter(parameterDescriptor, value, null, true)); + parameters.add(new Parameter.Builder() + .name(key) + .value(value) + .provided(true) + .build()); }); parameterGroups.add(new ParameterGroup(secretName, parameters)); } diff --git a/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultParameterProvider.java b/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultParameterProvider.java index 9880448c676c..5973da39288e 100644 --- a/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultParameterProvider.java +++ b/nifi-extension-bundles/nifi-hashicorp-vault-bundle/nifi-hashicorp-vault-parameter-provider/src/test/java/org/apache/nifi/vault/hashicorp/TestHashiCorpVaultParameterProvider.java @@ -22,7 +22,6 @@ import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.logging.ComponentLog; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -157,6 +156,9 @@ private void mockSecrets(final String kvPath, final List paramet } private Parameter createParameter(final String name, final String value) { - return new Parameter(new ParameterDescriptor.Builder().name(name).build(), value); + return new Parameter.Builder() + .name(name) + .value(value) + .build(); } } diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/DatabaseParameterProvider.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/DatabaseParameterProvider.java index bd66a99f101d..c2f429316f9c 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/DatabaseParameterProvider.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/DatabaseParameterProvider.java @@ -207,11 +207,11 @@ public List fetchParameters(final ConfigurationContext context) parameterGroupName = tableName; } - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder() - .name(parameterName) - .build(); - final Parameter parameter = new Parameter(parameterDescriptor, parameterValue); - + final Parameter parameter = new Parameter.Builder() + .name(parameterName) + .value(parameterValue) + .provided(true) + .build(); parameterMap.computeIfAbsent(parameterGroupName, key -> new ArrayList<>()).add(parameter); } } diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/EnvironmentVariableParameterProvider.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/EnvironmentVariableParameterProvider.java index b6066ae9dfc9..8e89a04a2b03 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/EnvironmentVariableParameterProvider.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/EnvironmentVariableParameterProvider.java @@ -150,8 +150,11 @@ public List fetchParameters(final ConfigurationContext context) environmentVariables .forEach( (key, value) -> { if (inclusionStrategy.include(key)) { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(key).build(); - parameters.add(new Parameter(parameterDescriptor, value, null, true)); + parameters.add(new Parameter.Builder() + .name(key) + .value(value) + .provided(true) + .build()); } }); return Collections.singletonList(new ParameterGroup(parameterGroupName, parameters)); diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/KubernetesSecretParameterProvider.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/KubernetesSecretParameterProvider.java index 67a6e75ac13a..a5826dcf3935 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/KubernetesSecretParameterProvider.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/KubernetesSecretParameterProvider.java @@ -203,8 +203,11 @@ private ParameterGroup getParameterGroup(final ConfigurationContext context, fin getLogger().warn("Parameter {} may be truncated at {} bytes", parameterName, parameterValue.length()); } - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(parameterName).build(); - parameters.add(new Parameter(parameterDescriptor, parameterValue, null, true)); + parameters.add(new Parameter.Builder() + .name(parameterName) + .value(parameterValue) + .provided(true) + .build()); } catch (final IOException e) { throw new RuntimeException(String.format("Failed to read file [%s]", file), e); } diff --git a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/OnePasswordParameterProvider.java b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/OnePasswordParameterProvider.java index f78053954bf3..6b9deb6c14b1 100644 --- a/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/OnePasswordParameterProvider.java +++ b/nifi-extension-bundles/nifi-standard-bundle/nifi-standard-parameter-providers/src/main/java/org/apache/nifi/parameter/OnePasswordParameterProvider.java @@ -216,12 +216,13 @@ public List fetchParameters(final ConfigurationContext context) final JsonNode fieldValue = field.get("value"); if (fieldValue != null) { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(itemName + "_" + fieldId).build(); - parameters.add(new Parameter(parameterDescriptor, fieldValue.asText(), null, true)); + parameters.add(new Parameter.Builder() + .name(itemName + "_" + fieldId) + .value(fieldValue.asText()) + .provided(true) + .build()); } - } - } parameterGroups.add(new ParameterGroup(vaultName, parameters)); diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManager.java b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManager.java new file mode 100644 index 000000000000..d1501c2f9335 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManager.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Optional; + +public interface AssetManager { + /** + * Initializes the AssetManager, providing the context necessary for the manager to operate. + * @param context the context that provides all necessary initialization information + */ + void initialize(AssetManagerInitializationContext context); + + /** + * Creates a new Asset with the given name and contents. + * @param parameterContextId the id of the parameter context + * @param assetName the name of the asset + * @param contents the contents of the asset + * @return the created asset + * @throws IOException if there is an error creating the asset + */ + Asset createAsset(String parameterContextId, String assetName, InputStream contents) throws IOException; + + /** + * Retrieves the Asset with the given id, if it exists. + * @param id the id of the asset to retrieve + * @return the asset, if it exists + */ + Optional getAsset(String id); + + /** + * Retrieves the Assets that belong to the given parameter context. + * @param parameterContextId the id of the parameter context + * @return the list of assets for the given context + */ + List getAssets(String parameterContextId); + + /** + * Creates an Asset with the given name and associates it with the given parameter context. If the asset already exists, it is returned. Otherwise, an asset is created + * but the underlying file is not created. This allows the asset to be referenced but any component that attempts to use the asset will still see a File that does not exist, which + * will typically lead to an invalid component. + * + * @param parameterContextId the id of the parameter context + * @param assetName the name of the asset + * @return the created asset + */ + Asset createMissingAsset(String parameterContextId, String assetName); + + /** + * Deletes the Asset with the given id, if it exists. + * @param id the id of the asset to delete + * @return the deleted asset, if it existed + */ + Optional deleteAsset(String id); +} diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManagerInitializationContext.java b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManagerInitializationContext.java new file mode 100644 index 000000000000..5357a6a1f580 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetManagerInitializationContext.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.controller.NodeTypeProvider; + +import java.util.Map; + +/** + * Initialization context for AssetManager. + */ +public interface AssetManagerInitializationContext { + + /** + * @return lookup for obtaining referenced assets + */ + AssetReferenceLookup getAssetReferenceLookup(); + + /** + * @return configuration for the asset manager + */ + Map getProperties(); + + /** + * @return the node type provider + */ + NodeTypeProvider getNodeTypeProvider(); +} diff --git a/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetReferenceLookup.java b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetReferenceLookup.java new file mode 100644 index 000000000000..38990dc78967 --- /dev/null +++ b/nifi-framework-api/src/main/java/org/apache/nifi/asset/AssetReferenceLookup.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import java.util.Set; + +public interface AssetReferenceLookup { + + /** + * The Set of all Assets that are currently referenced by any parameters. + */ + Set getReferencedAssets(); + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetDTO.java new file mode 100644 index 000000000000..49d11c786c2a --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetDTO.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +import java.util.Objects; + +@XmlType(name = "asset") +public class AssetDTO { + + private String id; + private String name; + private String digest; + private Boolean missingContent; + + @Schema(description = "The identifier of the asset.") + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + @Schema(description = "The name of the asset.") + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @Schema(description = "The digest of the asset, will be null if the asset content is missing.") + public String getDigest() { + return digest; + } + + public void setDigest(final String digest) { + this.digest = digest; + } + + @Schema(description = "Indicates if the content of the asset is missing.") + public Boolean getMissingContent() { + return missingContent; + } + + public void setMissingContent(final Boolean missingContent) { + this.missingContent = missingContent; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AssetDTO assetDTO = (AssetDTO) o; + return Objects.equals(id, assetDTO.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetReferenceDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetReferenceDTO.java new file mode 100644 index 000000000000..8db542277c9b --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/AssetReferenceDTO.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.web.api.dto; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlType; + +import java.util.Objects; + +@XmlType(name = "assetReference") +public class AssetReferenceDTO { + + private String id; + private String name; + + public AssetReferenceDTO() { + } + + public AssetReferenceDTO(final String id) { + this.id = id; + } + + public AssetReferenceDTO(final String id, final String name) { + this.id = id; + this.name = name; + } + + @Schema(description = "The identifier of the referenced asset.") + public String getId() { + return id; + } + + public void setId(final String id) { + this.id = id; + } + + @Schema(description = "The name of the referenced asset.", accessMode = Schema.AccessMode.READ_ONLY) + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AssetReferenceDTO that = (AssetReferenceDTO) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hashCode(id); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java index c50857da9a4f..92102192a0c9 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/dto/ParameterDTO.java @@ -21,6 +21,8 @@ import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import jakarta.xml.bind.annotation.XmlType; + +import java.util.List; import java.util.Set; @XmlType(name = "parameter") @@ -31,6 +33,7 @@ public class ParameterDTO { private String value; private Boolean valueRemoved; private Boolean provided; + private List referencedAssets; private Set referencingComponents; private ParameterContextReferenceEntity parameterContext; private Boolean inherited; @@ -122,8 +125,17 @@ public void setReferencingComponents(final Set referenc this.referencingComponents = referencingComponents; } + @Schema(description = "A list of identifiers of the assets that are referenced by the parameter") + public List getReferencedAssets() { + return referencedAssets; + } + + public void setReferencedAssets(final List referencedAssets) { + this.referencedAssets = referencedAssets; + } + @Override public String toString() { - return "ParameterDTO[name=" + name + ", sensitive=" + sensitive + ", value=" + (sensitive ? "********" : value) + (provided ? " (provided)" : "") + "]"; + return "ParameterDTO[name=" + name + ", sensitive=" + sensitive + ", value=" + (sensitive == Boolean.TRUE ? "********" : value) + (provided == Boolean.TRUE ? " (provided)" : "") + "]"; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetEntity.java new file mode 100644 index 000000000000..f05c3b61ea00 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetEntity.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; +import org.apache.nifi.web.api.dto.AssetDTO; + +import java.util.Objects; + +@XmlRootElement(name = "assetEntity") +public class AssetEntity extends Entity { + private AssetDTO asset; + + @Schema(description = "The Asset.", + accessMode = Schema.AccessMode.READ_ONLY + ) + public AssetDTO getAsset() { + return asset; + } + + public void setAsset(AssetDTO asset) { + this.asset = asset; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AssetEntity that = (AssetEntity) o; + return Objects.equals(asset, that.asset); + } + + @Override + public int hashCode() { + return Objects.hash(asset); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetsEntity.java b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetsEntity.java new file mode 100644 index 000000000000..c48ee1dab01e --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-client-dto/src/main/java/org/apache/nifi/web/api/entity/AssetsEntity.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.web.api.entity; + +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.xml.bind.annotation.XmlRootElement; + +import java.util.Collection; + +@XmlRootElement(name = "assetEntity") +public class AssetsEntity extends Entity { + + private Collection assets; + + @Schema(description = "The asset entities") + public Collection getAssets() { + return assets; + } + + public void setAssets(final Collection assets) { + this.assets = assets; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java index e3ae59a9bcf4..fb7d2ba0bad7 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/StandardHttpResponseMapper.java @@ -18,6 +18,7 @@ import jakarta.ws.rs.core.StreamingOutput; import org.apache.nifi.cluster.coordination.http.endpoints.AccessPolicyEndpointMerger; +import org.apache.nifi.cluster.coordination.http.endpoints.AssetsEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.BulletinBoardEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ComponentStateEndpointMerger; import org.apache.nifi.cluster.coordination.http.endpoints.ConnectionEndpointMerger; @@ -201,7 +202,7 @@ public StandardHttpResponseMapper(final NiFiProperties nifiProperties) { endpointMergers.add(new NarSummaryEndpointMerger()); endpointMergers.add(new NarSummariesEndpointMerger()); endpointMergers.add(new NarDetailsEndpointMerger()); - + endpointMergers.add(new AssetsEndpointMerger()); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AssetsEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AssetsEndpointMerger.java new file mode 100644 index 000000000000..422ade1492d0 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/AssetsEndpointMerger.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.coordination.http.endpoints; + +import org.apache.nifi.cluster.manager.AssetsEntityMerger; +import org.apache.nifi.cluster.manager.NodeResponse; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; + +import java.net.URI; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class AssetsEndpointMerger extends AbstractSingleDTOEndpoint> { + + private static final Pattern ASSETS_URI = Pattern.compile("/nifi-api/parameter-contexts/[a-f0-9\\-]{36}/assets"); + + @Override + public boolean canHandle(final URI uri, final String method) { + return "GET".equalsIgnoreCase(method) && ASSETS_URI.matcher(uri.getPath()).matches(); + } + + @Override + protected Class getEntityClass() { + return AssetsEntity.class; + } + + @Override + protected Collection getDto(final AssetsEntity entity) { + return entity.getAssets(); + } + + @Override + protected void mergeResponses(final Collection clientDto, final Map> dtoMap, + final Set successfulResponses, final Set problematicResponses) { + AssetsEntityMerger.mergeResponses(clientDto, dtoMap); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ParameterContextUpdateEndpointMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ParameterContextUpdateEndpointMerger.java index 2d7de0d2ab98..2b885c6502ee 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ParameterContextUpdateEndpointMerger.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/coordination/http/endpoints/ParameterContextUpdateEndpointMerger.java @@ -33,13 +33,12 @@ import java.util.regex.Pattern; public class ParameterContextUpdateEndpointMerger extends AbstractSingleEntityEndpoint implements EndpointResponseMerger { - private static final Pattern PARAMETER_CONTEXT_URI = Pattern.compile("/nifi-api/parameter-contexts/[a-f0-9\\-]{36}"); - private static final String PARAMETER_CONTEXTS_URI = "/nifi-api/parameter-contexts"; + private static final Pattern PARAMETER_CONTEXT_UPDATE_REQUEST_URI = Pattern.compile("/nifi-api/parameter-contexts/[a-f0-9\\-]{36}/update-requests(/[a-f0-9\\-]{36})?"); @Override public boolean canHandle(final URI uri, final String method) { - return ("GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method)) && PARAMETER_CONTEXT_URI.matcher(uri.getPath()).matches() - || "POST".equalsIgnoreCase(method) && PARAMETER_CONTEXTS_URI.equals(method); + return ("GET".equalsIgnoreCase(method) || "DELETE".equalsIgnoreCase(method) || "POST".equalsIgnoreCase(method)) + && PARAMETER_CONTEXT_UPDATE_REQUEST_URI.matcher(uri.getPath()).matches(); } @Override diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/AssetsEntityMerger.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/AssetsEntityMerger.java new file mode 100644 index 000000000000..c32157eb1586 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/main/java/org/apache/nifi/cluster/manager/AssetsEntityMerger.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.entity.AssetEntity; + +import java.util.Collection; +import java.util.Map; + +public class AssetsEntityMerger { + + public static void mergeResponses(final Collection clientDto, final Map> dtoMap) { + dtoMap.values().forEach(clientDto::retainAll); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/AssetsEntityMergerTest.java b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/AssetsEntityMergerTest.java new file mode 100644 index 000000000000..635f8c201457 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-cluster/src/test/java/org/apache/nifi/cluster/manager/AssetsEntityMergerTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.cluster.manager; + +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.web.api.dto.AssetDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class AssetsEntityMergerTest { + + private static final String ASSET_ID_FORMAT = "1234/%s"; + private static final String ASSET_FILENAME_1 = "one.txt"; + private static final String ASSET_FILENAME_2 = "two.txt"; + private static final String ASSET_FILENAME_3 = "three.txt"; + private static final String ASSET_FILENAME_4 = "four.txt"; + + @Test + public void testMergeAssetEntities() { + final AssetEntity assetEntity1 = createAssetEntity(ASSET_ID_FORMAT.formatted(ASSET_FILENAME_1), ASSET_FILENAME_1); + final AssetEntity assetEntity2 = createAssetEntity(ASSET_ID_FORMAT.formatted(ASSET_FILENAME_2), ASSET_FILENAME_2); + final AssetEntity assetEntity3 = createAssetEntity(ASSET_ID_FORMAT.formatted(ASSET_FILENAME_3), ASSET_FILENAME_3); + final AssetEntity assetEntity4 = createAssetEntity(ASSET_ID_FORMAT.formatted(ASSET_FILENAME_4), ASSET_FILENAME_4); + + final Collection mergedResults = new ArrayList<>(); + mergedResults.add(assetEntity1); + mergedResults.add(assetEntity2); + mergedResults.add(assetEntity3); + + final NodeIdentifier node1Identifier = new NodeIdentifier("node1", "localhost", 8080, "localhost", 8081, "localhost", 8082, 8083, false); + final Collection node1Response = new ArrayList<>(); + node1Response.add(assetEntity2); + node1Response.add(assetEntity3); + + final NodeIdentifier node2Identifier = new NodeIdentifier("node2", "localhost", 8080, "localhost", 8081, "localhost", 8082, 8083, false); + final Collection node2Response = new ArrayList<>(); + node2Response.add(assetEntity1); + node2Response.add(assetEntity3); + node2Response.add(assetEntity4); + + final Map> nodeMap = Map.of( + node1Identifier, node1Response, + node2Identifier, node2Response + ); + + AssetsEntityMerger.mergeResponses(mergedResults, nodeMap); + + assertEquals(1, mergedResults.size()); + assertEquals(assetEntity3, mergedResults.stream().findFirst().orElse(null)); + } + + private AssetEntity createAssetEntity(final String id, final String name) { + final AssetDTO assetDTO = new AssetDTO(); + assetDTO.setId(id); + assetDTO.setName(name); + + final AssetEntity assetEntity = new AssetEntity(); + assetEntity.setAsset(assetDTO); + return assetEntity; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAsset.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAsset.java new file mode 100644 index 000000000000..77e2ef5b64de --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAsset.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import java.io.File; +import java.util.Objects; +import java.util.Optional; + +public class StandardAsset implements Asset { + private final String identifier; + private final String parameterContextIdentifier; + private final String name; + private final File file; + private final String digest; + + public StandardAsset(final String identifier, final String paramContextIdentifier, final String name, final File file, final String digest) { + this.identifier = Objects.requireNonNull(identifier, "Identifier is required"); + this.parameterContextIdentifier = Objects.requireNonNull(paramContextIdentifier, "Parameter Context Identifier is required"); + this.name = Objects.requireNonNull(name, "Name is required"); + this.file = Objects.requireNonNull(file, "File is required"); + this.digest = digest; + } + + @Override + public String getIdentifier() { + return identifier; + } + + @Override + public String getParameterContextIdentifier() { + return parameterContextIdentifier; + } + + @Override + public String getName() { + return name; + } + + @Override + public File getFile() { + return file; + } + + @Override + public Optional getDigest() { + return Optional.ofNullable(digest); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManager.java new file mode 100644 index 000000000000..f643cccf3a73 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManager.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.nar.FileDigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +public class StandardAssetManager implements AssetManager { + private static final Logger logger = LoggerFactory.getLogger(StandardAssetManager.class); + + public static final String ASSET_STORAGE_LOCATION_PROPERTY = "directory"; + public static final String DEFAULT_ASSET_STORAGE_LOCATION = "./assets"; + + private volatile File assetStorageLocation; + private final Map assets = new ConcurrentHashMap<>(); + + @Override + public void initialize(final AssetManagerInitializationContext context) { + final String storageLocation = getStorageLocation(context); + + assetStorageLocation = new File(storageLocation); + if (!assetStorageLocation.exists()) { + try { + Files.createDirectories(assetStorageLocation.toPath()); + } catch (IOException e) { + throw new RuntimeException("The Asset Manager's [%s] property is set to [%s] but the directory does not exist and cannot be created" + .formatted(ASSET_STORAGE_LOCATION_PROPERTY, storageLocation), e); + } + } + + try { + recoverLocalAssets(); + } catch (final IOException e) { + throw new RuntimeException("Unable to access assets", e); + } + } + + @Override + public Asset createAsset(final String parameterContextId, final String assetName, final InputStream contents) throws IOException { + final String assetId = createAssetId(parameterContextId, assetName); + + final File file = getFile(parameterContextId, assetName); + final File dir = file.getParentFile(); + if (!dir.exists()) { + try { + Files.createDirectories(dir.toPath()); + } catch (final IOException ioe) { + throw new IOException("Could not create directory in order to store asset", ioe); + } + } + + // Write contents to a temporary file, then move it to the final location. + // This allows us to avoid a situation where we upload a file, then we attempt to overwrite it but fail, leaving a corrupt asset. + final File tempFile = new File(dir, file.getName() + ".tmp"); + logger.debug("Writing temp asset file [{}]", tempFile.getAbsolutePath()); + + try { + Files.copy(contents, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + } catch (final Exception e) { + throw new IOException("Failed to write asset to file " + tempFile.getAbsolutePath(), e); + } + + Files.move(tempFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + + final String digest = computeDigest(file); + final Asset asset = new StandardAsset(assetId, parameterContextId, assetName, file, digest); + assets.put(assetId, asset); + return asset; + } + + @Override + public Optional getAsset(final String id) { + return Optional.ofNullable(assets.get(id)); + } + + @Override + public List getAssets(final String parameterContextId) { + final List allAssets = new ArrayList<>(assets.values()); + final List paramContextAssets = new ArrayList<>(); + for (final Asset asset : allAssets) { + if (asset.getParameterContextIdentifier().equals(parameterContextId)) { + paramContextAssets.add(asset); + } + } + return paramContextAssets; + } + + @Override + public Asset createMissingAsset(final String parameterContextId, final String assetName) { + final String assetId = createAssetId(parameterContextId, assetName); + final File file = getFile(parameterContextId, assetName); + final Asset asset = new StandardAsset(assetId, parameterContextId, assetName, file, null); + assets.put(assetId, asset); + return asset; + } + + @Override + public Optional deleteAsset(final String id) { + final Asset removed = assets.remove(id); + if (removed == null) { + return Optional.empty(); + } + + final File file = removed.getFile(); + if (file.exists()) { + try { + Files.delete(file.toPath()); + } catch (final IOException e) { + logger.warn("Failed to remove asset file {}", file.getAbsolutePath(), e); + } + + final File parentDir = file.getParentFile(); + final File[] children = parentDir.listFiles(); + if (children != null && children.length == 0) { + try { + Files.delete(parentDir.toPath()); + } catch (IOException e) { + logger.warn("Failed to remove empty asset directory {}", parentDir.getAbsolutePath(), e); + } + } + } + + return Optional.of(removed); + } + + private String createAssetId(final String parameterContextId, final String assetName) { + final String seed = parameterContextId + "/" + assetName; + return UUID.nameUUIDFromBytes(seed.getBytes(StandardCharsets.UTF_8)).toString(); + } + + private File getFile(final String paramContextId, final String assetName) { + final Path parentPath = assetStorageLocation.toPath().normalize(); + final Path assetPath = Paths.get(paramContextId, assetName).normalize(); + final Path fullPath = parentPath.resolve(assetPath); + return fullPath.toFile(); + } + + private String getStorageLocation(final AssetManagerInitializationContext initializationContext) { + final String storageLocation = initializationContext.getProperties().get(ASSET_STORAGE_LOCATION_PROPERTY); + return storageLocation == null ? DEFAULT_ASSET_STORAGE_LOCATION : storageLocation; + } + + private void recoverLocalAssets() throws IOException { + final File[] files = assetStorageLocation.listFiles(); + if (files == null) { + throw new IOException("Unable to list files for asset storage location %s".formatted(assetStorageLocation.getAbsolutePath())); + } + + for (final File file : files) { + if (!file.isDirectory()) { + continue; + } + + final String contextId = file.getName(); + final File[] assetFiles = file.listFiles(); + if (assetFiles == null) { + logger.warn("Unable to determine which assets exist for Parameter Context {}", contextId); + continue; + } + + for (final File assetFile : assetFiles) { + final String assetId = createAssetId(contextId, assetFile.getName()); + final String digest = computeDigest(assetFile); + final Asset asset = new StandardAsset(assetId, contextId, assetFile.getName(), assetFile, digest); + assets.put(assetId, asset); + } + } + } + + private String computeDigest(final File file) throws IOException { + return HexFormat.of().formatHex(FileDigestUtils.getDigest(file)); + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManagerInitializationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManagerInitializationContext.java new file mode 100644 index 000000000000..dcefe0b92639 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetManagerInitializationContext.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.controller.NodeTypeProvider; + +import java.util.Map; +import java.util.Objects; + +public class StandardAssetManagerInitializationContext implements AssetManagerInitializationContext { + private final AssetReferenceLookup assetReferenceLookup; + private final Map properties; + private final NodeTypeProvider nodeTypeProvider; + + public StandardAssetManagerInitializationContext(final AssetReferenceLookup assetReferenceLookup, final Map properties, final NodeTypeProvider nodeTypeProvider) { + this.assetReferenceLookup = Objects.requireNonNull(assetReferenceLookup); + this.properties = Objects.requireNonNull(properties); + this.nodeTypeProvider = Objects.requireNonNull(nodeTypeProvider); + } + + @Override + public AssetReferenceLookup getAssetReferenceLookup() { + return assetReferenceLookup; + } + + @Override + public Map getProperties() { + return properties; + } + + @Override + public NodeTypeProvider getNodeTypeProvider() { + return nodeTypeProvider; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetReferenceLookup.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetReferenceLookup.java new file mode 100644 index 000000000000..29c31cf5dd80 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/asset/StandardAssetReferenceLookup.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterContextManager; + +import java.util.HashSet; +import java.util.Set; + +public class StandardAssetReferenceLookup implements AssetReferenceLookup { + private final ParameterContextManager parameterContextManager; + + public StandardAssetReferenceLookup(final ParameterContextManager parameterContextManager) { + this.parameterContextManager = parameterContextManager; + } + + @Override + public Set getReferencedAssets() { + final Set assets = new HashSet<>(); + + for (final ParameterContext context : parameterContextManager.getParameterContexts()) { + for (final Parameter parameter : context.getParameters().values()) { + assets.addAll(parameter.getReferencedAssets()); + } + } + + return assets; + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java index 2d3330da940d..7c8e8963b820 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/flowanalysis/FlowAnalysisUtil.java @@ -26,17 +26,17 @@ public class FlowAnalysisUtil { public static final String ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE = "*****"; public static NiFiRegistryFlowMapper createMapper(ExtensionManager extensionManager) { - NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper( - extensionManager, - new FlowMappingOptions.Builder() - .mapPropertyDescriptors(true) - .mapControllerServiceReferencesToVersionedId(true) - .stateLookup(VersionedComponentStateLookup.IDENTITY_LOOKUP) - .componentIdLookup(ComponentIdLookup.USE_COMPONENT_ID) - .mapSensitiveConfiguration(true) - .sensitiveValueEncryptor(value -> ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE) - .build() - ) { + final FlowMappingOptions flowMappingOptions = new FlowMappingOptions.Builder() + .mapPropertyDescriptors(true) + .mapControllerServiceReferencesToVersionedId(true) + .stateLookup(VersionedComponentStateLookup.IDENTITY_LOOKUP) + .componentIdLookup(ComponentIdLookup.USE_COMPONENT_ID) + .mapSensitiveConfiguration(true) + .sensitiveValueEncryptor(value -> ENCRYPTED_SENSITIVE_VALUE_SUBSTITUTE) + .mapAssetReferences(true) + .build(); + + final NiFiRegistryFlowMapper mapper = new NiFiRegistryFlowMapper(extensionManager, flowMappingOptions) { @Override public String getGroupId(String groupId) { return groupId; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/parameter/StandardParameterProviderNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/parameter/StandardParameterProviderNode.java index 479e782cb925..e4d6a073f6d4 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/parameter/StandardParameterProviderNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/controller/parameter/StandardParameterProviderNode.java @@ -542,12 +542,12 @@ private List configureParameters(final Collection paramete throw new IllegalArgumentException(String.format("Parameter sensitivity must be specified for parameter [%s] in group [%s]", parameterName, groupConfiguration.getGroupName())); } - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder() - .from(parameter.getDescriptor()) - .name(parameterName) - .sensitive(sensitivity == ParameterSensitivity.SENSITIVE) - .build(); - return new Parameter(parameterDescriptor, parameter.getValue(), parameter.getParameterContextId(), true); + + return new Parameter.Builder() + .fromParameter(parameter) + .sensitive(sensitivity == ParameterSensitivity.SENSITIVE) + .provided(true) + .build(); }) .collect(Collectors.toList()); } @@ -559,7 +559,7 @@ private List configureParameters(final Collection paramete */ private static List toProvidedParameters(final Collection parameters) { return parameters == null ? Collections.emptyList() : parameters.stream() - .map(parameter -> new Parameter(parameter.getDescriptor(), parameter.getValue(), null, true)) + .map(parameter -> new Parameter.Builder().descriptor(parameter.getDescriptor()).value(parameter.getValue()).provided(true).build()) .collect(Collectors.toList()); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java index dd4b40893e74..47aadddb6ba1 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/StandardVersionedComponentSynchronizer.java @@ -17,6 +17,8 @@ package org.apache.nifi.flow.synchronization; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.connectable.Connectable; @@ -53,6 +55,7 @@ import org.apache.nifi.flow.ConnectableComponentType; import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.flow.ParameterProviderReference; +import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; @@ -457,9 +460,9 @@ private void synchronize(final ProcessGroup group, final VersionedProcessGroup p restoreConnectionDestinations(group, proposed, connectionsByVersionedId, connectionsWithTempDestination); } - Map newParameters = new HashMap<>(); + final Map newParameters = new HashMap<>(); if (!proposedParameterContextExistsBeforeSynchronize && this.context.getFlowMappingOptions().isMapControllerServiceReferencesToVersionedId()) { - Map controllerServiceVersionedIdToId = group.getControllerServices(false) + final Map controllerServiceVersionedIdToId = group.getControllerServices(false) .stream() .filter(controllerServiceNode -> controllerServiceNode.getVersionedComponentId().isPresent()) .collect(Collectors.toMap( @@ -467,18 +470,22 @@ private void synchronize(final ProcessGroup group, final VersionedProcessGroup p ComponentNode::getIdentifier )); - ParameterContext parameterContext = group.getParameterContext(); + final ParameterContext parameterContext = group.getParameterContext(); if (parameterContext != null) { parameterContext.getParameters().forEach((descriptor, parameter) -> { - List referencedControllerServiceData = parameterContext + final List referencedControllerServiceData = parameterContext .getParameterReferenceManager() .getReferencedControllerServiceData(parameterContext, descriptor.getName()); if (referencedControllerServiceData.isEmpty()) { newParameters.put(descriptor.getName(), parameter); } else { - final Parameter adjustedParameter = new Parameter(parameter.getDescriptor(), controllerServiceVersionedIdToId.get(parameter.getValue())); + final Parameter adjustedParameter = new Parameter.Builder() + .fromParameter(parameter) + .value(controllerServiceVersionedIdToId.get(parameter.getValue())) + .build(); + newParameters.put(descriptor.getName(), adjustedParameter); } }); @@ -1677,15 +1684,15 @@ public void synchronize(final ParameterContext parameterContext, final Versioned } protected Set getUpdatedParameterNames(final ParameterContext parameterContext, final VersionedParameterContext proposed) { - final Map originalValues = new HashMap<>(); - parameterContext.getParameters().values().forEach(param -> originalValues.put(param.getDescriptor().getName(), param.getValue())); + final Map originalValues = new HashMap<>(); + parameterContext.getParameters().values().forEach(param -> originalValues.put(param.getDescriptor().getName(), getValueAndReferences(param))); - final Map proposedValues = new HashMap<>(); + final Map proposedValues = new HashMap<>(); if (proposed != null) { - proposed.getParameters().forEach(versionedParam -> proposedValues.put(versionedParam.getName(), versionedParam.getValue())); + proposed.getParameters().forEach(versionedParam -> proposedValues.put(versionedParam.getName(), getValueAndReferences(versionedParam))); } - final Map copyOfOriginalValues = new HashMap<>(originalValues); + final Map copyOfOriginalValues = new HashMap<>(originalValues); proposedValues.forEach(originalValues::remove); copyOfOriginalValues.forEach(proposedValues::remove); @@ -1695,6 +1702,24 @@ protected Set getUpdatedParameterNames(final ParameterContext parameterC return updatedParameterNames; } + private ParameterValueAndReferences getValueAndReferences(final Parameter parameter) { + final List assets = parameter.getReferencedAssets(); + if (assets == null || assets.isEmpty()) { + return new ParameterValueAndReferences(parameter.getValue(), null); + } + final List assetIds = assets.stream().map(Asset::getIdentifier).toList(); + return new ParameterValueAndReferences(null, assetIds); + } + + private ParameterValueAndReferences getValueAndReferences(final VersionedParameter parameter) { + final List assets = parameter.getReferencedAssets(); + if (assets == null || assets.isEmpty()) { + return new ParameterValueAndReferences(parameter.getValue(), null); + } + final List assetIds = assets.stream().map(VersionedAsset::getIdentifier).toList(); + return new ParameterValueAndReferences(null, assetIds); + } + @Override public void synchronizeProcessGroupSettings(final ProcessGroup processGroup, final VersionedProcessGroup proposed, final ProcessGroup parentGroup, final FlowSynchronizationOptions synchronizationOptions) @@ -2033,13 +2058,8 @@ private ParameterContext createParameterContextWithoutReferences(final Versioned if (versionedParameter == null) { continue; } - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() - .name(versionedParameter.getName()) - .description(versionedParameter.getDescription()) - .sensitive(versionedParameter.isSensitive()) - .build(); - final Parameter parameter = new Parameter(descriptor, versionedParameter.getValue(), null, versionedParameter.isProvided()); + final Parameter parameter = createParameter(null, versionedParameter); parameters.put(versionedParameter.getName(), parameter); } @@ -2068,8 +2088,8 @@ private ParameterContext createParameterContext(final VersionedParameterContext final AtomicReference contextReference = new AtomicReference<>(); context.getFlowManager().withParameterContextResolution(() -> { final ParameterContext created = context.getFlowManager().createParameterContext(parameterContextId, versionedParameterContext.getName(), - versionedParameterContext.getDescription(), parameters, parameterContextRefs, - getParameterProviderConfiguration(versionedParameterContext)); + versionedParameterContext.getDescription(), parameters, parameterContextRefs, getParameterProviderConfiguration(versionedParameterContext)); + contextReference.set(created); }); @@ -2079,13 +2099,7 @@ private ParameterContext createParameterContext(final VersionedParameterContext private Map createParameterMap(final Collection versionedParameters) { final Map parameters = new HashMap<>(); for (final VersionedParameter versionedParameter : versionedParameters) { - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() - .name(versionedParameter.getName()) - .description(versionedParameter.getDescription()) - .sensitive(versionedParameter.isSensitive()) - .build(); - - final Parameter parameter = new Parameter(descriptor, versionedParameter.getValue(), null, versionedParameter.isProvided()); + final Parameter parameter = createParameter(null, versionedParameter); parameters.put(versionedParameter.getName(), parameter); } @@ -2129,13 +2143,7 @@ private void addMissingConfiguration(final VersionedParameterContext versionedPa continue; } - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() - .name(versionedParameter.getName()) - .description(versionedParameter.getDescription()) - .sensitive(versionedParameter.isSensitive()) - .build(); - - final Parameter parameter = new Parameter(descriptor, versionedParameter.getValue(), null, versionedParameter.isProvided()); + final Parameter parameter = createParameter(currentParameterContext.getIdentifier(), versionedParameter); parameters.put(versionedParameter.getName(), parameter); } @@ -2149,12 +2157,40 @@ private void addMissingConfiguration(final VersionedParameterContext versionedPa .map(name -> selectParameterContext(versionedParameterContexts.get(name), versionedParameterContexts, parameterProviderReferences, componentIdGenerator)) .collect(Collectors.toList())); } + if (versionedParameterContext.getParameterProvider() != null && currentParameterContext.getParameterProvider() == null) { createMissingParameterProvider(versionedParameterContext, versionedParameterContext.getParameterProvider(), parameterProviderReferences, componentIdGenerator); currentParameterContext.configureParameterProvider(getParameterProviderConfiguration(versionedParameterContext)); } } + private Parameter createParameter(final String contextId, final VersionedParameter versionedParameter) { + final List referencedAssets = versionedParameter.getReferencedAssets(); + + final List assets; + if (referencedAssets == null || referencedAssets.isEmpty()) { + assets = null; + } else { + final AssetManager assetManager = context.getAssetManager(); + assets = new ArrayList<>(); + for (final VersionedAsset reference : referencedAssets) { + final Optional assetOption = assetManager.getAsset(reference.getIdentifier()); + final Asset asset = assetOption.orElseGet(() -> assetManager.createMissingAsset(contextId, reference.getName())); + assets.add(asset); + } + } + + return new Parameter.Builder() + .name(versionedParameter.getName()) + .description(versionedParameter.getDescription()) + .sensitive(versionedParameter.isSensitive()) + .value(versionedParameter.getValue()) + .referencedAssets(assets) + .provided(versionedParameter.isProvided()) + .parameterContextId(contextId) + .build(); + } + private boolean isEqual(final BundleCoordinate coordinate, final Bundle bundle) { if (!bundle.getGroup().equals(coordinate.getGroup())) { return false; @@ -3769,4 +3805,6 @@ private ControllerServiceNode getVersionedControllerService(final ProcessGroup g private record CreatedExtension(ComponentNode extension, Map propertyValues) { } + + private record ParameterValueAndReferences(String value, List assetIds) { } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedFlowSynchronizationContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedFlowSynchronizationContext.java index 29c233b3ae7a..12734dcd2b41 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedFlowSynchronizationContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/flow/synchronization/VersionedFlowSynchronizationContext.java @@ -17,6 +17,7 @@ package org.apache.nifi.flow.synchronization; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.controller.ProcessorNode; @@ -43,6 +44,8 @@ public class VersionedFlowSynchronizationContext { private final FlowMappingOptions flowMappingOptions; private final Function processContextFactory; private final Function configurationContextFactory; + private final AssetManager assetManager; + private VersionedFlowSynchronizationContext(final Builder builder) { this.componentIdGenerator = builder.componentIdGenerator; @@ -54,6 +57,7 @@ private VersionedFlowSynchronizationContext(final Builder builder) { this.flowMappingOptions = builder.flowMappingOptions; this.processContextFactory = builder.processContextFactory; this.configurationContextFactory = builder.configurationContextFactory; + this.assetManager = builder.assetManager; } public ComponentIdGenerator getComponentIdGenerator() { @@ -92,6 +96,10 @@ public Function getConfigurationContextFact return configurationContextFactory; } + public AssetManager getAssetManager() { + return assetManager; + } + public static class Builder { private ComponentIdGenerator componentIdGenerator; private FlowManager flowManager; @@ -102,6 +110,8 @@ public static class Builder { private FlowMappingOptions flowMappingOptions; private Function processContextFactory; private Function configurationContextFactory; + private AssetManager assetManager; + public Builder componentIdGenerator(final ComponentIdGenerator componentIdGenerator) { this.componentIdGenerator = componentIdGenerator; @@ -148,6 +158,11 @@ public Builder configurationContextFactory(final Function(); position = new AtomicReference<>(new Position(0D, 0D)); @@ -281,14 +285,6 @@ public ProcessGroup getParent() { return parent.get(); } - private ProcessGroup getRoot() { - ProcessGroup root = this; - while (root.getParent() != null) { - root = root.getParent(); - } - return root; - } - @Override public void setParent(final ProcessGroup newParent) { parent.set(newParent); @@ -3838,6 +3834,7 @@ public void updateFlow(final VersionedExternalFlow proposedSnapshot, final Strin .mapInstanceIdentifiers(false) .mapControllerServiceReferencesToVersionedId(true) .mapFlowRegistryClientId(false) + .mapAssetReferences(false) .build(); synchronizeFlow(proposedSnapshot, synchronizationOptions, flowMappingOptions); @@ -4022,7 +4019,8 @@ private VersionedFlowSynchronizationContext createGroupSynchronizationContext(fi .componentScheduler(componentScheduler) .flowMappingOptions(flowMappingOptions) .processContextFactory(this::createProcessContext) - .configurationContextFactory(this::createConfigurationContext) + .configurationContextFactory(this::createConfigurationContext) + .assetManager(assetManager) .build(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java index a16f54f4085d..da8f4cad032c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java @@ -241,14 +241,12 @@ private Map updateParameters(final Map> overrideParameters(final Map

new ArrayList<>()).add(overridingParameter); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java index 766518a8c57e..48c7eed7896f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/java/org/apache/nifi/registry/flow/mapping/NiFiRegistryFlowMapper.java @@ -18,6 +18,7 @@ package org.apache.nifi.registry.flow.mapping; import org.apache.commons.lang3.ClassUtils; +import org.apache.nifi.asset.Asset; import org.apache.nifi.bundle.BundleCoordinate; import org.apache.nifi.components.PropertyDescriptor; import org.apache.nifi.components.resource.ResourceCardinality; @@ -50,6 +51,7 @@ import org.apache.nifi.flow.ParameterProviderReference; import org.apache.nifi.flow.PortType; import org.apache.nifi.flow.Position; +import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedFlowAnalysisRule; @@ -964,11 +966,26 @@ private VersionedParameter mapParameter(final Parameter parameter, final String versionedParameter.setSensitive(descriptor.isSensitive()); versionedParameter.setProvided(parameter.isProvided()); - final String mapped = parameterValueMapper.getMapped(parameter, value); - versionedParameter.setValue(mapped); + final List referencedAssets = parameter.getReferencedAssets(); + if (referencedAssets == null || referencedAssets.isEmpty()) { + final String mapped = parameterValueMapper.getMapped(parameter, value); + versionedParameter.setValue(mapped); + } else if (flowMappingOptions.isMapAssetReferences()) { + final List assetIds = referencedAssets.stream() + .map(this::createVersionedAsset) + .toList(); + versionedParameter.setReferencedAssets(assetIds); + } return versionedParameter; } + private VersionedAsset createVersionedAsset(final Asset asset) { + final VersionedAsset versionedAsset = new VersionedAsset(); + versionedAsset.setIdentifier(asset.getIdentifier()); + versionedAsset.setName(asset.getName()); + return versionedAsset; + } + private org.apache.nifi.flow.ScheduledState mapScheduledState(final ScheduledState scheduledState) { return scheduledState == ScheduledState.DISABLED ? org.apache.nifi.flow.ScheduledState.DISABLED diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/resources/META-INF/services/org.apache.nifi.asset.AssetManager b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/resources/META-INF/services/org.apache.nifi.asset.AssetManager new file mode 100644 index 000000000000..6fbf05d94972 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/main/resources/META-INF/services/org.apache.nifi.asset.AssetManager @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.asset.StandardAssetManager \ No newline at end of file diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java index 328c63f88a56..9e83dab33d26 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterContext.java @@ -58,8 +58,8 @@ public void testUpdatesApply() { final ParameterDescriptor fooDescriptor = new ParameterDescriptor.Builder().name("foo").description("bar").sensitive(true).build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); - parameters.put("xyz", new Parameter(xyzDescriptor, "242526")); + parameters.put("abc", createParameter(abcDescriptor, "123")); + parameters.put("xyz", createParameter(xyzDescriptor, "242526")); context.setParameters(parameters); @@ -74,7 +74,7 @@ public void testUpdatesApply() { assertEquals("242526", xyzParam.getValue()); final Map secondParameters = new HashMap<>(); - secondParameters.put("foo", new Parameter(fooDescriptor, "baz")); + secondParameters.put("foo", createParameter(fooDescriptor, "baz")); context.setParameters(secondParameters); assertTrue(context.getParameter("abc").isPresent()); @@ -97,12 +97,20 @@ public void testUpdatesApply() { assertEquals(Collections.singletonMap(fooDescriptor, fooParam), context.getParameters()); final Map thirdParameters = new HashMap<>(); - thirdParameters.put("foo", new Parameter(fooDescriptor, "other")); + thirdParameters.put("foo", createParameter(fooDescriptor, "other")); context.setParameters(thirdParameters); assertEquals("other", context.getParameter("foo").get().getValue()); } + private static Parameter createParameter(final ParameterDescriptor descriptor, final String value) { + return createParameter(descriptor, value, false); + } + + private static Parameter createParameter(final ParameterDescriptor descriptor, final String value, final boolean provided) { + return new Parameter.Builder().descriptor(descriptor).value(value).provided(provided).build(); + } + @Test public void testUpdateDescription() { final ParameterReferenceManager referenceManager = new HashMapParameterReferenceManager(); @@ -114,7 +122,7 @@ public void testUpdateDescription() { final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").description("abc").build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); + parameters.put("abc", createParameter(abcDescriptor, "123")); context.setParameters(parameters); @@ -124,7 +132,7 @@ public void testUpdateDescription() { assertEquals("123", abcParam.getValue()); ParameterDescriptor updatedDescriptor = new ParameterDescriptor.Builder().name("abc").description("Updated").build(); - final Parameter newDescriptionParam = new Parameter(updatedDescriptor, "321"); + final Parameter newDescriptionParam = createParameter(updatedDescriptor, "321"); context.setParameters(Collections.singletonMap("abc", newDescriptionParam)); abcParam = context.getParameter("abc").get(); @@ -133,7 +141,7 @@ public void testUpdateDescription() { assertEquals("321", abcParam.getValue()); updatedDescriptor = new ParameterDescriptor.Builder().name("abc").description("Updated Again").build(); - final Parameter paramWithoutValue = new Parameter(updatedDescriptor, null); + final Parameter paramWithoutValue = createParameter(updatedDescriptor, null); context.setParameters(Collections.singletonMap("abc", paramWithoutValue)); abcParam = context.getParameter("abc").get(); @@ -153,7 +161,7 @@ public void testUpdateSensitivity() { final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").description("abc").build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123", null, true)); + parameters.put("abc", createParameter(abcDescriptor, "123", true)); context.setParameters(parameters); @@ -163,10 +171,10 @@ public void testUpdateSensitivity() { assertEquals("123", abcParam.getValue()); ParameterDescriptor updatedDescriptor = new ParameterDescriptor.Builder().name("abc").description("abc").sensitive(true).build(); - final Parameter unprovidedParam = new Parameter(updatedDescriptor, "321", null, false); + final Parameter unprovidedParam = createParameter(updatedDescriptor, "321", false); assertThrows(IllegalStateException.class, () -> context.setParameters(Collections.singletonMap("abc", unprovidedParam))); - final Parameter newSensitivityParam = new Parameter(updatedDescriptor, "321", null, true); + final Parameter newSensitivityParam = createParameter(updatedDescriptor, "321", true); context.setParameters(Collections.singletonMap("abc", newSensitivityParam)); abcParam = context.getParameter("abc").get(); @@ -188,19 +196,19 @@ public void testChangeDescription() { .build(); final ParameterDescriptor xyzDescriptor = new ParameterDescriptor.Builder().name("xyz").build(); final Map parameters = new HashMap<>(); - parameters.put("xyz", new Parameter(xyzDescriptor, "123")); + parameters.put("xyz", createParameter(xyzDescriptor, "123")); context.setParameters(parameters); final Map updates = new HashMap<>(); final ParameterDescriptor xyzDescriptor2 = new ParameterDescriptor.Builder().from(xyzDescriptor).description("changed").build(); - final Parameter updatedParameter = new Parameter(xyzDescriptor2, "123"); + final Parameter updatedParameter = createParameter(xyzDescriptor2, "123"); updates.put("xyz", updatedParameter); assertEquals(1, context.getEffectiveParameterUpdates(updates, Collections.emptyList()).size()); // Now there is no change, since the description is the same final Map updates2 = new HashMap<>(); final ParameterDescriptor xyzDescriptor3 = new ParameterDescriptor.Builder().from(xyzDescriptor).description("changed").build(); - final Parameter updatedParameter2 = new Parameter(xyzDescriptor3, "123"); + final Parameter updatedParameter2 = createParameter(xyzDescriptor3, "123"); updates.put("xyz", updatedParameter2); assertEquals(0, context.getEffectiveParameterUpdates(updates2, Collections.emptyList()).size()); } @@ -219,23 +227,23 @@ public void testChangingSensitivity() { final ParameterDescriptor fooDescriptor = new ParameterDescriptor.Builder().name("foo").description("bar").sensitive(true).build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); - parameters.put("xyz", new Parameter(xyzDescriptor, "242526")); + parameters.put("abc", createParameter(abcDescriptor, "123")); + parameters.put("xyz", createParameter(xyzDescriptor, "242526")); context.setParameters(parameters); final ParameterDescriptor sensitiveXyzDescriptor = new ParameterDescriptor.Builder().name("xyz").sensitive(true).build(); final Map updatedParameters = new HashMap<>(); - updatedParameters.put("foo", new Parameter(fooDescriptor, "baz")); - updatedParameters.put("xyz", new Parameter(sensitiveXyzDescriptor, "242526")); + updatedParameters.put("foo", createParameter(fooDescriptor, "baz")); + updatedParameters.put("xyz", createParameter(sensitiveXyzDescriptor, "242526")); assertThrows(IllegalStateException.class, () -> context.setParameters(updatedParameters)); final ParameterDescriptor insensitiveAbcDescriptor = new ParameterDescriptor.Builder().name("abc").sensitive(false).build(); updatedParameters.clear(); - updatedParameters.put("abc", new Parameter(insensitiveAbcDescriptor, "123")); + updatedParameters.put("abc", createParameter(insensitiveAbcDescriptor, "123")); assertThrows(IllegalStateException.class, () -> context.setParameters(updatedParameters)); @@ -250,12 +258,12 @@ public void testChangingParameterForRunningProcessor() { final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").sensitive(true).build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); + parameters.put("abc", createParameter(abcDescriptor, "123")); context.setParameters(parameters); parameters.clear(); - parameters.put("abc", new Parameter(abcDescriptor, "321")); + parameters.put("abc", createParameter(abcDescriptor, "321")); context.setParameters(parameters); assertEquals("321", context.getParameter("abc").get().getValue()); @@ -264,7 +272,7 @@ public void testChangingParameterForRunningProcessor() { startProcessor(procNode); parameters.clear(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); + parameters.put("abc", createParameter(abcDescriptor, "123")); // Cannot update parameters while running assertThrows(IllegalStateException.class, () -> context.setParameters(parameters)); @@ -273,7 +281,7 @@ public void testChangingParameterForRunningProcessor() { context.setParameters(Collections.emptyMap()); parameters.clear(); - parameters.put("abc", new Parameter(abcDescriptor, null)); + parameters.put("abc", createParameter(abcDescriptor, null)); assertThrows(IllegalStateException.class, () -> context.setParameters(parameters)); @@ -474,14 +482,14 @@ public void testChangingParameterForEnabledControllerService() { final ParameterDescriptor abcDescriptor = new ParameterDescriptor.Builder().name("abc").sensitive(true).build(); final Map parameters = new HashMap<>(); - parameters.put("abc", new Parameter(abcDescriptor, "123")); + parameters.put("abc", createParameter(abcDescriptor, "123")); context.setParameters(parameters); referenceManager.addControllerServiceReference("abc", serviceNode); parameters.clear(); - parameters.put("abc", new Parameter(abcDescriptor, "321")); + parameters.put("abc", createParameter(abcDescriptor, "321")); for (final ControllerServiceState state : EnumSet.of(ControllerServiceState.ENABLED, ControllerServiceState.ENABLING, ControllerServiceState.DISABLING)) { setControllerServiceState(serviceNode, state); @@ -498,7 +506,7 @@ public void testChangingParameterForEnabledControllerService() { parameters.clear(); context.setParameters(parameters); - parameters.put("abc", new Parameter(abcDescriptor, null)); + parameters.put("abc", createParameter(abcDescriptor, null)); try { context.setParameters(parameters); fail("Was able to remove parameter being referenced by Controller Service that is DISABLING"); @@ -765,7 +773,7 @@ private static ParameterDescriptor addParameter(final ParameterContext parameter parameters.put(entry.getKey().getName(), entry.getValue()); } final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder().name(name).sensitive(isSensitive).build(); - parameters.put(name, new Parameter(parameterDescriptor, value)); + parameters.put(name, createParameter(parameterDescriptor, value)); parameterContext.setParameters(parameters); return parameterDescriptor; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterValueMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterValueMapper.java index 6667ced1c221..eb4669313576 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterValueMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-components/src/test/java/org/apache/nifi/parameter/TestStandardParameterValueMapper.java @@ -91,6 +91,6 @@ void testGetMappedSensitiveNotProvidedNullValue() { private Parameter getParameter(final boolean sensitive, final boolean provided) { final ParameterDescriptor descriptor = new ParameterDescriptor.Builder().name(NAME).sensitive(sensitive).build(); - return new Parameter(descriptor, VALUE, null, provided); + return new Parameter.Builder().descriptor(descriptor).value(VALUE).provided(provided).build(); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetComponentManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetComponentManager.java new file mode 100644 index 000000000000..35ff55a71600 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetComponentManager.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +/** + * Manages restarting components which reference a given Asset through a Parameter. + */ +public interface AssetComponentManager { + + /** + * Asynchronously restarts any components referencing the given asset. + * + * @param asset the asset + */ + void restartReferencingComponentsAsync(Asset asset); + + /** + * Synchronously restarts any components referencing the given asset. + * + * @param asset the asset + */ + void restartReferencingComponents(Asset asset); + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetSynchronizer.java new file mode 100644 index 000000000000..7556e7b6cd82 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/asset/AssetSynchronizer.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +/** + * Responsible for synchronizing assets from the cluster to the current node. + */ +public interface AssetSynchronizer { + + /** + * Synchronizes the current node's assets + */ + void synchronize(); + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java index dc188e223d3b..91ae0ea278d0 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/controller/AbstractComponentNode.java @@ -1306,7 +1306,12 @@ public Optional getParameter(final String parameterName) { isProvided = false; } - final Parameter updatedParameter = new Parameter(parameterDescriptor, parameterUpdate.getPreviousValue(), null, isProvided); + final Parameter updatedParameter = new Parameter.Builder() + .descriptor(parameterDescriptor) + .value(parameterUpdate.getPreviousValue()) + .provided(isProvided) + .build(); + return Optional.of(updatedParameter); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/FlowMappingOptions.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/FlowMappingOptions.java index 66be84a37392..8a5b24deb000 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/FlowMappingOptions.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/main/java/org/apache/nifi/registry/flow/mapping/FlowMappingOptions.java @@ -28,6 +28,8 @@ public class FlowMappingOptions { private final boolean mapInstanceIds; private final boolean mapControllerServiceReferencesToVersionedId; private final boolean mapFlowRegistryClientId; + private final boolean mapAssetReferences; + private FlowMappingOptions(final Builder builder) { encryptor = builder.encryptor; @@ -38,6 +40,7 @@ private FlowMappingOptions(final Builder builder) { mapInstanceIds = builder.mapInstanceId; mapControllerServiceReferencesToVersionedId = builder.mapControllerServiceReferencesToVersionedId; mapFlowRegistryClientId = builder.mapFlowRegistryClientId; + mapAssetReferences = builder.mapAssetReferences; } public SensitiveValueEncryptor getSensitiveValueEncryptor() { @@ -72,6 +75,10 @@ public boolean isMapFlowRegistryClientId() { return mapFlowRegistryClientId; } + public boolean isMapAssetReferences() { + return mapAssetReferences; + } + public static class Builder { private SensitiveValueEncryptor encryptor; private VersionedComponentStateLookup stateLookup; @@ -81,6 +88,7 @@ public static class Builder { private boolean mapInstanceId = false; private boolean mapControllerServiceReferencesToVersionedId = true; private boolean mapFlowRegistryClientId = false; + private boolean mapAssetReferences = false; /** * Sets the SensitiveValueEncryptor to use for encrypting sensitive values. This value must be set @@ -177,6 +185,11 @@ public Builder mapFlowRegistryClientId(final boolean mapFlowRegistryClientId) { return this; } + public Builder mapAssetReferences(final boolean mapAssetReferences) { + this.mapAssetReferences = mapAssetReferences; + return this; + } + /** * Creates a FlowMappingOptions object, or throws an Exception if not all required configuration has been provided * @@ -210,6 +223,7 @@ public FlowMappingOptions build() { .mapInstanceIdentifiers(false) .mapControllerServiceReferencesToVersionedId(true) .mapFlowRegistryClientId(false) + .mapAssetReferences(false) .build(); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestAbstractComponentNode.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestAbstractComponentNode.java index 2b4abbc10dee..e4beebf4ffb3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestAbstractComponentNode.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core-api/src/test/java/org/apache/nifi/controller/TestAbstractComponentNode.java @@ -116,7 +116,7 @@ protected void onPropertyModified(final PropertyDescriptor descriptor, final Str .description("") .sensitive(false) .build(); - final Parameter param = new Parameter(paramDescriptor, "123"); + final Parameter param = new Parameter.Builder().descriptor(paramDescriptor).value("123").build(); Mockito.doReturn(Optional.of(param)).when(context).getParameter("abc"); node.setParameterContext(context); @@ -150,7 +150,7 @@ public void testMismatchedSensitiveFlags() { .description("") .sensitive(true) .build(); - final Parameter param = new Parameter(paramDescriptor, "123"); + final Parameter param = new Parameter.Builder().descriptor(paramDescriptor).value("123").build(); Mockito.doReturn(Optional.of(param)).when(context).getParameter("abc"); node.setParameterContext(context); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml index 011f38205669..7b0487d7fa6d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/pom.xml @@ -317,6 +317,7 @@ org.glassfish.jaxb jaxb-runtime + org.apache.curator curator-test diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/AssetsRestApiClient.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/AssetsRestApiClient.java new file mode 100644 index 000000000000..b53a2387a2d3 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/AssetsRestApiClient.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.client.NiFiRestApiClient; +import org.apache.nifi.client.NiFiRestApiRetryableException; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.client.StandardHttpUriBuilder; +import org.apache.nifi.web.client.api.HttpRequestBodySpec; +import org.apache.nifi.web.client.api.HttpResponseEntity; +import org.apache.nifi.web.client.api.WebClientService; +import org.apache.nifi.web.client.api.WebClientServiceException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.InputStream; +import java.net.URI; + +/** + * Encapsulates the REST API calls required to synchronize assets. + */ +class AssetsRestApiClient extends NiFiRestApiClient { + + private static final Logger logger = LoggerFactory.getLogger(AssetsRestApiClient.class); + + private static final String NIFI_API_PATH_PART = "nifi-api"; + private static final String PARAMETER_CONTEXTS_PATH_PART = "parameter-contexts"; + private static final String ASSETS_PATH_PART = "assets"; + + public AssetsRestApiClient(final WebClientService webClientService, final String host, final int port, final boolean secure) { + super(webClientService, host, port, secure); + } + + public AssetsEntity getAssets(final String parameterContextId) { + final URI requestUri = new StandardHttpUriBuilder() + .scheme(baseUri.getScheme()) + .host(baseUri.getHost()) + .port(baseUri.getPort()) + .addPathSegment(NIFI_API_PATH_PART) + .addPathSegment(PARAMETER_CONTEXTS_PATH_PART) + .addPathSegment(parameterContextId) + .addPathSegment(ASSETS_PATH_PART) + .build(); + logger.debug("Requesting Asset listing from {}", requestUri); + + // Send the replicated header so that the cluster coordinator does not replicate the request, otherwise this call can happen when no nodes + // are considered connected and result in a 500 exception that can't easily be differentiated from other unknown errors + final HttpRequestBodySpec requestBodySpec = webClientService.get() + .uri(requestUri) + .header(ACCEPT_HEADER, APPLICATION_JSON) + .header(X_REQUEST_REPLICATED_HEADER, "true"); + + return executeEntityRequest(requestUri, requestBodySpec, AssetsEntity.class); + } + + public InputStream getAssetContent(final String parameterContextId, final String assetId) { + final URI requestUri = new StandardHttpUriBuilder() + .scheme(baseUri.getScheme()) + .host(baseUri.getHost()) + .port(baseUri.getPort()) + .addPathSegment(NIFI_API_PATH_PART) + .addPathSegment(PARAMETER_CONTEXTS_PATH_PART) + .addPathSegment(parameterContextId) + .addPathSegment(ASSETS_PATH_PART) + .addPathSegment(assetId) + .build(); + logger.debug("Getting asset content from {}", requestUri); + + try { + final HttpResponseEntity response = webClientService.get() + .uri(requestUri) + .header(ACCEPT_HEADER, APPLICATION_OCTET_STREAM) + .retrieve(); + return getResponseBody(requestUri, response); + } catch (final WebClientServiceException e) { + throw new NiFiRestApiRetryableException(e.getMessage(), e); + } + } + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetComponentManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetComponentManager.java new file mode 100644 index 000000000000..78b197cfb00f --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetComponentManager.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.nifi.controller.ComponentNode; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.ProcessorNode; +import org.apache.nifi.controller.ScheduledState; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.controller.service.ControllerServiceNode; +import org.apache.nifi.controller.service.ControllerServiceProvider; +import org.apache.nifi.flow.ExecutionEngine; +import org.apache.nifi.groups.ProcessGroup; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterContextManager; +import org.apache.nifi.parameter.ParameterReferenceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +public class StandardAssetComponentManager implements AssetComponentManager { + + private static final Logger logger = LoggerFactory.getLogger(StandardAssetComponentManager.class); + private static final Duration COMPONENT_WAIT_TIMEOUT = Duration.ofSeconds(30); + + private final FlowManager flowManager; + private final ControllerServiceProvider controllerServiceProvider; + + public StandardAssetComponentManager(final FlowController flowController) { + this.flowManager = flowController.getFlowManager(); + this.controllerServiceProvider = flowController.getControllerServiceProvider(); + } + + @Override + public void restartReferencingComponentsAsync(final Asset asset) { + final String assetName = asset.getName(); + final String paramContextId = asset.getParameterContextIdentifier(); + Thread.ofVirtual().name("Restart Components for Asset [%s] in ParameterContext [%s]".formatted(assetName, paramContextId)).start(() -> { + try { + restartReferencingComponents(asset); + } catch (final Exception e) { + logger.error("Failed to restart referencing components for Asset [{}] in ParameterContext [{}]", assetName, paramContextId); + } + }); + } + + @Override + public void restartReferencingComponents(final Asset asset) { + final ParameterContextManager parameterContextManager = flowManager.getParameterContextManager(); + final ParameterContext parameterContext = parameterContextManager.getParameterContext(asset.getParameterContextIdentifier()); + + // Determine which parameters reference the given asset + final Set parametersReferencingAsset = getParametersReferencingAsset(parameterContext, asset); + if (parametersReferencingAsset.isEmpty()) { + logger.info("Asset [{}] is not referenced by any parameters in ParameterContext [{}] ", asset.getName(), parameterContext.getIdentifier()); + return; + } + + // Determine processors, controller services, and stateless PGs that reference a parameter that references the asset + final Set processorsReferencingParameters = getProcessorsReferencingParameters(parameterContext, parametersReferencingAsset); + final Set controllerServicesReferencingParameters = getControllerServicesReferencingParameters(parameterContext, parametersReferencingAsset); + final Set affectedStatelessGroups = getAffectedStatelessGroups(parameterContext); + + logger.info("Found {} stateless groups, {} processors, and {} controller services referencing Asset [{}]", affectedStatelessGroups.size(), + processorsReferencingParameters.size(), controllerServicesReferencingParameters.size(), asset.getName()); + + // Stop/disable the impacted components + final Set stoppedStatelessProcessGroups = new HashSet<>(); + affectedStatelessGroups.forEach(processGroup -> stopProcessGroup(processGroup, stoppedStatelessProcessGroups)); + + final Set stoppedProcessors = new HashSet<>(); + processorsReferencingParameters.forEach(processorNode -> stopProcessors(processorNode, stoppedProcessors)); + + final Set disabledControllerServices = new HashSet<>(); + controllerServicesReferencingParameters.forEach(controllerServiceNode -> disableControllerService(controllerServiceNode, disabledControllerServices, stoppedProcessors)); + + // Start/enable the components that were previously stopped/disabled + enableControllerServices(disabledControllerServices); + stoppedProcessors.forEach(this::startProcessor); + stoppedStatelessProcessGroups.forEach(ProcessGroup::startProcessing); + } + + private void startProcessor(final ProcessorNode processorNode) { + try { + final Future future = processorNode.getProcessGroup().startProcessor(processorNode, false); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to start processor within the given time period", e); + } + } + + private void enableControllerServices(final Set controllerServices) { + try { + final Future future = controllerServiceProvider.enableControllerServicesAsync(controllerServices); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to enable controller services within the given time period", e); + } + } + + private void stopProcessors(final ProcessorNode processorNode, final Set stoppedProcessors) { + if (!processorNode.isRunning() && processorNode.getPhysicalScheduledState() != ScheduledState.STARTING) { + return; + } + + try { + final Future future = processorNode.getProcessGroup().stopProcessor(processorNode); + stoppedProcessors.add(processorNode); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to stop processor [{}], processor will be terminated", processorNode.getIdentifier(), e); + processorNode.terminate(); + } + } + + private void disableControllerService(final ControllerServiceNode controllerServiceNode, final Set disabledControllerServices, + final Set stoppedProcessors) { + if (!controllerServiceNode.isActive()) { + return; + } + + // Unscheduled components that reference the current controller service + final Map> futures = controllerServiceProvider.unscheduleReferencingComponents(controllerServiceNode); + for (final Map.Entry> entry : futures.entrySet()) { + final ComponentNode component = entry.getKey(); + if (component instanceof ProcessorNode processorNode) { + stoppedProcessors.add(processorNode); + } else { + logger.warn("Unexpected stopped component of type {} with ID {}}", component.getCanonicalClassName(), component.getIdentifier()); + } + + try { + final Future future = entry.getValue(); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to unschedule referencing component [{}]", component.getIdentifier(), e); + } + } + + // Find other controller services that are enabled and reference the current controller service + final List referencingServices = controllerServiceNode.getReferences().findRecursiveReferences(ControllerServiceNode.class).stream() + .filter(ControllerServiceNode::isActive) + .toList(); + + // Disable the current service and the referencing services + final Set servicesToDisable = new HashSet<>(); + servicesToDisable.add(controllerServiceNode); + servicesToDisable.addAll(referencingServices); + + try { + final Future future = controllerServiceProvider.disableControllerServicesAsync(servicesToDisable); + disabledControllerServices.addAll(servicesToDisable); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to disable controller service [{}], or one of it's referencing services", controllerServiceNode.getIdentifier(), e); + } + } + + private void stopProcessGroup(final ProcessGroup processGroup, final Set stoppedProcessGroups) { + if (!processGroup.isStatelessActive()) { + return; + } + + try { + final Future future = processGroup.stopProcessing(); + stoppedProcessGroups.add(processGroup); + future.get(COMPONENT_WAIT_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); + } catch (final Exception e) { + logger.warn("Failed to stop process group [{}]", processGroup.getIdentifier(), e); + } + } + + private Set getProcessorsReferencingParameters(final ParameterContext parameterContext, final Set parameters) { + final ParameterReferenceManager parameterReferenceManager = parameterContext.getParameterReferenceManager(); + final Set allReferencingProcessors = new HashSet<>(); + for (final Parameter parameter : parameters) { + final String parameterName = parameter.getDescriptor().getName(); + final Set referencingProcessors = parameterReferenceManager.getProcessorsReferencing(parameterContext, parameterName); + allReferencingProcessors.addAll(referencingProcessors); + } + return allReferencingProcessors; + } + + private Set getControllerServicesReferencingParameters(final ParameterContext parameterContext, final Set parameters) { + final ParameterReferenceManager parameterReferenceManager = parameterContext.getParameterReferenceManager(); + + final Set allReferencingServices = new HashSet<>(); + for (final Parameter parameter : parameters) { + final String parameterName = parameter.getDescriptor().getName(); + final Set referencingServices = parameterReferenceManager.getControllerServicesReferencing(parameterContext, parameterName); + allReferencingServices.addAll(referencingServices); + } + return allReferencingServices; + } + + private Set getAffectedStatelessGroups(final ParameterContext parameterContext) { + final ParameterReferenceManager parameterReferenceManager = parameterContext.getParameterReferenceManager(); + final Set boundProcessGroups = parameterReferenceManager.getProcessGroupsBound(parameterContext); + + final Set affectedStatelessGroups = new HashSet<>(); + for (final ProcessGroup processGroup : boundProcessGroups) { + final ProcessGroup statelessGroup = getStatelessParent(processGroup); + if (statelessGroup != null && statelessGroup.isStatelessActive()) { + affectedStatelessGroups.add(statelessGroup); + } + } + return affectedStatelessGroups; + } + + private ProcessGroup getStatelessParent(final ProcessGroup group) { + if (group == null) { + return null; + } + + final ExecutionEngine engine = group.getExecutionEngine(); + if (engine == ExecutionEngine.STATELESS) { + return group; + } + + return getStatelessParent(group.getParent()); + } + + private Set getParametersReferencingAsset(final ParameterContext parameterContext, final Asset asset) { + final Set referencingParameters = new HashSet<>(); + for (final Parameter parameter : parameterContext.getParameters().values()) { + for (final Asset parameterAsset : parameter.getReferencedAssets()) { + if (parameterAsset.getIdentifier().equals(asset.getIdentifier())) { + referencingParameters.add(parameter); + } + } + } + return referencingParameters; + } + +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetSynchronizer.java new file mode 100644 index 000000000000..a64fef129da2 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/asset/StandardAssetSynchronizer.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.asset; + +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.nifi.client.NiFiRestApiRetryableException; +import org.apache.nifi.cluster.coordination.ClusterCoordinator; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.controller.FlowController; +import org.apache.nifi.controller.flow.FlowManager; +import org.apache.nifi.parameter.Parameter; +import org.apache.nifi.parameter.ParameterContext; +import org.apache.nifi.parameter.ParameterContextManager; +import org.apache.nifi.util.NiFiProperties; +import org.apache.nifi.web.api.dto.AssetDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.client.api.WebClientService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Collection; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Synchronizes assets from the cluster coordinator using the NiFi REST API. + */ +public class StandardAssetSynchronizer implements AssetSynchronizer { + + private static final Logger logger = LoggerFactory.getLogger(StandardAssetSynchronizer.class); + + private static final Duration CLUSTER_COORDINATOR_RETRY_DURATION = Duration.ofSeconds(60); + private static final Duration LIST_ASSETS_RETRY_DURATION = Duration.ofMinutes(5); + private static final Duration SYNC_ASSET_RETRY_DURATION = Duration.ofSeconds(30); + + private final AssetManager assetManager; + private final FlowManager flowManager; + private final ClusterCoordinator clusterCoordinator; + private final WebClientService webClientService; + private final NiFiProperties properties; + + public StandardAssetSynchronizer(final FlowController flowController, + final ClusterCoordinator clusterCoordinator, + final WebClientService webClientService, + final NiFiProperties properties) { + this.assetManager = flowController.getAssetManager(); + this.flowManager = flowController.getFlowManager(); + this.clusterCoordinator = clusterCoordinator; + this.webClientService = webClientService; + this.properties = properties; + } + + @Override + public void synchronize() { + if (clusterCoordinator == null) { + logger.info("Clustering is not configured: Asset synchronization disabled"); + return; + } + + // This sync method is called from the method that loads the flow from a connection response, which means there must already be a cluster coordinator to + // have gotten a response from, but during testing there were cases where calling clusterCoordinator.getElectedActiveCoordinatorNode() was still null, so + // the helper method here will keep checking for the identifier up to a certain threshold to avoid slight timing issues + final NodeIdentifier coordinatorNodeId = getElectedActiveCoordinatorNode(); + if (coordinatorNodeId == null) { + logger.warn("Unable to obtain the node identifier for the cluster coordinator: Asset synchronization disabled"); + return; + } + + if (clusterCoordinator.isActiveClusterCoordinator()) { + logger.info("Current node is the cluster coordinator: Asset synchronization disabled"); + return; + } + + final String coordinatorAddress = coordinatorNodeId.getApiAddress(); + final int coordinatorPort = coordinatorNodeId.getApiPort(); + final AssetsRestApiClient assetsRestApiClient = new AssetsRestApiClient(webClientService, coordinatorAddress, coordinatorPort, properties.isHTTPSConfigured()); + logger.info("Synchronizing assets with cluster coordinator at {}:{}", coordinatorAddress, coordinatorPort); + + final ParameterContextManager parameterContextManager = flowManager.getParameterContextManager(); + final Set parameterContexts = parameterContextManager.getParameterContexts(); + logger.info("Found {} parameter contexts for synchronizing assets", parameterContexts.size()); + + for (final ParameterContext parameterContext : parameterContexts) { + try { + synchronize(assetsRestApiClient, parameterContext); + } catch (final Exception e) { + logger.error("Failed to synchronize assets for parameter context [{}]", parameterContext.getIdentifier(), e); + } + } + } + + private void synchronize(final AssetsRestApiClient assetsRestApiClient, final ParameterContext parameterContext) { + final Map existingAssets = parameterContext.getParameters().values().stream() + .map(Parameter::getReferencedAssets) + .flatMap(Collection::stream) + .collect(Collectors.toMap(Asset::getIdentifier, Function.identity())); + + if (existingAssets.isEmpty()) { + logger.info("Parameter context [{}] does not contain any assets to synchronize", parameterContext.getIdentifier()); + return; + } + + logger.info("Parameter context [{}] has {} assets on the current node", parameterContext.getIdentifier(), existingAssets.size()); + + final AssetsEntity coordinatorAssetsEntity = listAssetsWithRetry(assetsRestApiClient, parameterContext.getIdentifier()); + if (coordinatorAssetsEntity == null) { + logger.error("Timeout listing assets from cluster coordinator for parameter context [{}]", parameterContext.getIdentifier()); + return; + } + + final Collection coordinatorAssets = coordinatorAssetsEntity.getAssets(); + if (coordinatorAssets == null || coordinatorAssets.isEmpty()) { + logger.info("Parameter context [{}] did not return any assets from the cluster coordinator", parameterContext.getIdentifier()); + return; + } + + logger.info("Parameter context [{}] returned {} assets from the cluster coordinator", parameterContext.getIdentifier(), coordinatorAssets.size()); + + for (final AssetEntity coordinatorAssetEntity : coordinatorAssets) { + final AssetDTO coordinatorAsset = coordinatorAssetEntity.getAsset(); + final Asset matchingAsset = existingAssets.get(coordinatorAsset.getId()); + try { + synchronize(assetsRestApiClient, parameterContext, coordinatorAsset, matchingAsset); + } catch (final Exception e) { + logger.error("Failed to synchronize asset [id={},name={}] for parameter context [{}]", + coordinatorAsset.getId(), coordinatorAsset.getName(), parameterContext.getIdentifier(), e); + } + } + } + + private void synchronize(final AssetsRestApiClient assetsRestApiClient, final ParameterContext parameterContext, final AssetDTO coordinatorAsset, final Asset matchingAsset) { + final String paramContextId = parameterContext.getIdentifier(); + final String assetId = coordinatorAsset.getId(); + final String assetName = coordinatorAsset.getName(); + if (matchingAsset == null || !matchingAsset.getFile().exists()) { + logger.info("Synchronizing missing asset [id={},name={}] for parameter context [{}]", assetId, assetName, paramContextId); + synchronizeAssetWithRetry(assetsRestApiClient, paramContextId, coordinatorAsset); + } else { + final String coordinatorAssetDigest = coordinatorAsset.getDigest(); + final String matchingAssetDigest = matchingAsset.getDigest().orElse(null); + if (!coordinatorAssetDigest.equals(matchingAssetDigest)) { + logger.info("Synchronizing asset [id={},name={}] with updated digest [{}] for parameter context [{}]", + assetId, assetName, coordinatorAssetDigest, paramContextId); + synchronizeAssetWithRetry(assetsRestApiClient, paramContextId, coordinatorAsset); + } else { + logger.info("Coordinator asset [id={},name={}] found for parameter context [{}]: retrieval not required", assetId, assetName, paramContextId); + } + } + } + + private AssetsEntity listAssetsWithRetry(final AssetsRestApiClient assetsRestApiClient, final String parameterContextId) { + final Instant expirationTime = Instant.ofEpochMilli(System.currentTimeMillis() + LIST_ASSETS_RETRY_DURATION.toMillis()); + while (System.currentTimeMillis() < expirationTime.toEpochMilli()) { + final AssetsEntity assetsEntity = listAssets(assetsRestApiClient, parameterContextId); + if (assetsEntity != null) { + return assetsEntity; + } + logger.info("Unable to list assets from cluster coordinator for parameter context [{}]: retrying until [{}]", parameterContextId, expirationTime); + sleep(Duration.ofSeconds(5)); + } + return null; + } + + private AssetsEntity listAssets(final AssetsRestApiClient assetsRestApiClient, final String parameterContextId) { + try { + return assetsRestApiClient.getAssets(parameterContextId); + } catch (final NiFiRestApiRetryableException e) { + final Throwable rootCause = ExceptionUtils.getRootCause(e); + logger.warn("{}, root cause [{}]: retrying", e.getMessage(), rootCause.getMessage()); + return null; + } + } + + private void synchronizeAssetWithRetry(final AssetsRestApiClient assetsRestApiClient, final String parameterContextId, final AssetDTO coordinatorAsset) { + final Instant expirationTime = Instant.ofEpochMilli(System.currentTimeMillis() + SYNC_ASSET_RETRY_DURATION.toMillis()); + while (System.currentTimeMillis() < expirationTime.toEpochMilli()) { + final Asset syncedAsset = synchronizeAsset(assetsRestApiClient, parameterContextId, coordinatorAsset); + if (syncedAsset != null) { + return; + } + logger.info("Unable to synchronize asset [id={},name={}] for parameter context [{}]: retrying until [{}]", + coordinatorAsset.getId(), coordinatorAsset.getName(), parameterContextId, expirationTime); + sleep(Duration.ofSeconds(5)); + } + } + + private Asset synchronizeAsset(final AssetsRestApiClient assetsRestApiClient, final String parameterContextId, final AssetDTO coordinatorAsset) { + final String assetId = coordinatorAsset.getId(); + final String assetName = coordinatorAsset.getName(); + if (Boolean.TRUE == coordinatorAsset.getMissingContent()) { + return assetManager.createMissingAsset(parameterContextId, assetName); + } else { + try (final InputStream assetInputStream = assetsRestApiClient.getAssetContent(parameterContextId, assetId)) { + return assetManager.createAsset(parameterContextId, assetName, assetInputStream); + } catch (final NiFiRestApiRetryableException e) { + final Throwable rootCause = ExceptionUtils.getRootCause(e); + logger.warn("{}, root cause [{}]: retrying", e.getMessage(), rootCause.getMessage()); + return null; + } catch (final IOException e) { + final Throwable rootCause = ExceptionUtils.getRootCause(e); + logger.warn("Asset Manager failed to create asset [id={},name={}], root cause [{}]: retrying", assetId, assetId, rootCause.getMessage()); + return null; + } + } + } + + private NodeIdentifier getElectedActiveCoordinatorNode() { + final Instant expirationTime = Instant.ofEpochMilli(System.currentTimeMillis() + CLUSTER_COORDINATOR_RETRY_DURATION.toMillis()); + while (System.currentTimeMillis() < expirationTime.toEpochMilli()) { + final NodeIdentifier coordinatorNodeId = clusterCoordinator.getElectedActiveCoordinatorNode(); + if (coordinatorNodeId != null) { + return coordinatorNodeId; + } + logger.info("Node identifier for the active cluster coordinator is not known yet: retrying until [{}]", expirationTime); + sleep(Duration.ofSeconds(2)); + } + return null; + } + + private void sleep(final Duration duration) { + try { + Thread.sleep(duration); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiClient.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiClient.java new file mode 100644 index 000000000000..18171eef6903 --- /dev/null +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiClient.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.module.jakarta.xmlbind.JakartaXmlBindAnnotationIntrospector; +import org.apache.commons.io.IOUtils; +import org.apache.nifi.web.client.api.HttpRequestBodySpec; +import org.apache.nifi.web.client.api.HttpResponseEntity; +import org.apache.nifi.web.client.api.HttpResponseStatus; +import org.apache.nifi.web.client.api.WebClientService; +import org.apache.nifi.web.client.api.WebClientServiceException; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.Objects; + +/** + * Base class for calling the NiFi Rest API from framework code. + */ +public abstract class NiFiRestApiClient { + + private static final String HTTP_SCHEME = "http"; + private static final String HTTPS_SCHEME = "https"; + + protected static final String ACCEPT_HEADER = "Accept"; + protected static final String X_REQUEST_REPLICATED_HEADER = "X-Request-Replicated"; + protected static final String APPLICATION_JSON = "application/json"; + protected static final String APPLICATION_OCTET_STREAM = "application/octet-stream"; + + protected final URI baseUri; + protected final WebClientService webClientService; + protected final ObjectMapper objectMapper; + + /** + * @param webClientService the web client service provided by the framework + * @param host the base host + * @param port the base port + * @param secure indicates if the API is secure + */ + public NiFiRestApiClient(final WebClientService webClientService, final String host, final int port, final boolean secure) { + try { + this.baseUri = new URI(secure ? HTTPS_SCHEME : HTTP_SCHEME, null, host, port, null, null, null); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + this.webClientService = Objects.requireNonNull(webClientService, "WebClientService is required"); + this.objectMapper = new ObjectMapper(); + this.objectMapper.setAnnotationIntrospector(new JakartaXmlBindAnnotationIntrospector(objectMapper.getTypeFactory())); + } + + /** + * Executes the request specified by the given body spec and unmarshalls the response into the given entity, or throws an exception if the request was not successful. + * + * @param requestUri the URI being requested + * @param requestBodySpec the body spec + * @param responseEntityClass the entity class of the response + * @return an instance of the given entity class containing the response + * @param the type of response + */ + protected T executeEntityRequest(final URI requestUri, final HttpRequestBodySpec requestBodySpec, final Class responseEntityClass) { + try (final HttpResponseEntity response = requestBodySpec.retrieve()) { + final InputStream responseBody = getResponseBody(requestUri, response); + return objectMapper.readValue(responseBody, responseEntityClass); + } catch (final WebClientServiceException | IOException e) { + throw new NiFiRestApiRetryableException(e.getMessage(), e); + } + } + + /** + * Inspects the response and returns the body for a successful response, otherwise throws an exception based on the response code. + * + * @param requestUri the URI that was requested + * @param response the response + * @return the input stream of the response body + * @throws NiFiRestApiRetryableException if a retryable response or exception happens + * @throws IllegalStateException if a non-retryable response happens + */ + protected InputStream getResponseBody(final URI requestUri, final HttpResponseEntity response) { + final int statusCode = response.statusCode(); + if (HttpResponseStatus.OK.getCode() == statusCode) { + return response.body(); + } else { + final String responseMessage; + try { + responseMessage = IOUtils.toString(response.body(), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new NiFiRestApiRetryableException("Error reading response from %s - %s".formatted(requestUri, statusCode), e); + } + if (statusCode == HttpResponseStatus.CONFLICT.getCode()) { + throw new NiFiRestApiRetryableException("Error calling %s - %s - %s".formatted(requestUri, statusCode, responseMessage)); + } else { + throw new IllegalStateException("Error calling %s - %s - %s".formatted(requestUri, statusCode, responseMessage)); + } + } + } +} diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiRetryableException.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiRetryableException.java similarity index 72% rename from nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiRetryableException.java rename to nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiRetryableException.java index 83e722e2d90c..7518287a9387 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiRetryableException.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/client/NiFiRestApiRetryableException.java @@ -15,18 +15,18 @@ * limitations under the License. */ -package org.apache.nifi.nar; +package org.apache.nifi.client; /** - * Thrown from {@link NarRestApiClient} to indicate an error for a request that should be retried. + * Indicates a retryable error from calling NiFi's REST API. */ -public class NarRestApiRetryableException extends RuntimeException { +public class NiFiRestApiRetryableException extends RuntimeException { - public NarRestApiRetryableException(final String message) { + public NiFiRestApiRetryableException(final String message) { super(message); } - public NarRestApiRetryableException(final String message, final Throwable cause) { + public NiFiRestApiRetryableException(final String message, final Throwable cause) { super(message, cause); } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java index a0d6c7e7a348..e0dc7c88855c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/FlowController.java @@ -20,6 +20,13 @@ import org.apache.nifi.admin.service.AuditService; import org.apache.nifi.annotation.lifecycle.OnConfigurationRestored; import org.apache.nifi.annotation.notification.PrimaryNodeState; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.asset.AssetManagerInitializationContext; +import org.apache.nifi.asset.AssetReferenceLookup; +import org.apache.nifi.asset.StandardAssetManager; +import org.apache.nifi.asset.StandardAssetManagerInitializationContext; +import org.apache.nifi.asset.StandardAssetReferenceLookup; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.Resource; import org.apache.nifi.authorization.resource.Authorizable; @@ -255,6 +262,7 @@ public class FlowController implements ReportingTaskProvider, FlowAnalysisRulePr public static final String DEFAULT_CONTENT_REPO_IMPLEMENTATION = "org.apache.nifi.controller.repository.FileSystemRepository"; public static final String DEFAULT_PROVENANCE_REPO_IMPLEMENTATION = "org.apache.nifi.provenance.VolatileProvenanceRepository"; public static final String DEFAULT_SWAP_MANAGER_IMPLEMENTATION = "org.apache.nifi.controller.FileSystemSwapManager"; + public static final String DEFAULT_ASSET_MANAGER_IMPLEMENTATION = StandardAssetManager.class.getName(); public static final String GRACEFUL_SHUTDOWN_PERIOD = "nifi.flowcontroller.graceful.shutdown.seconds"; public static final long DEFAULT_GRACEFUL_SHUTDOWN_SECONDS = 10; @@ -268,6 +276,7 @@ public class FlowController implements ReportingTaskProvider, FlowAnalysisRulePr private final FlowFileEventRepository flowFileEventRepository; private final ProvenanceRepository provenanceRepository; private final BulletinRepository bulletinRepository; + private final AssetManager assetManager; private final LifecycleStateManager lifecycleStateManager; private final StandardProcessScheduler processScheduler; private final SnippetManager snippetManager; @@ -534,6 +543,7 @@ private FlowController( parameterContextManager = new StandardParameterContextManager(); repositoryContextFactory = new RepositoryContextFactory(contentRepository, flowFileRepository, flowFileEventRepository, counterRepositoryRef.get(), provenanceRepository, stateManagerProvider); + assetManager = createAssetManager(nifiProperties); this.flowAnalysisThreadPool = new FlowEngine(1, "Background Flow Analysis", true); if (ruleViolationsManager != null) { @@ -1339,6 +1349,78 @@ private ContentRepository createContentRepository(final NiFiProperties propertie } } + private AssetManager createAssetManager(final NiFiProperties properties) { + final String implementationClassName = properties.getProperty(NiFiProperties.ASSET_MANAGER_IMPLEMENTATION, DEFAULT_ASSET_MANAGER_IMPLEMENTATION); + + try { + final AssetManager assetManager = NarThreadContextClassLoader.createInstance(extensionManager, implementationClassName, AssetManager.class, properties); + final AssetReferenceLookup assetReferenceLookup = new StandardAssetReferenceLookup(parameterContextManager); + final Map relevantNiFiProperties = properties.getPropertiesWithPrefix(NiFiProperties.ASSET_MANAGER_PREFIX); + final int prefixLength = NiFiProperties.ASSET_MANAGER_PREFIX.length(); + final Map assetManagerProperties = relevantNiFiProperties.entrySet().stream() + .collect(Collectors.toMap(entry -> entry.getKey().substring(prefixLength), Map.Entry::getValue)); + + final AssetManagerInitializationContext initializationContext = new StandardAssetManagerInitializationContext( + assetReferenceLookup, assetManagerProperties, this); + + // Instrument Asset Manager with a wrapper that delegates to the appropriate class loader + final ClassLoader assetManagerClassLoader = assetManager.getClass().getClassLoader(); + final AssetManager instrumented = new AssetManager() { + @Override + public void initialize(final AssetManagerInitializationContext context) { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + assetManager.initialize(context); + } + } + + @Override + public Asset createAsset(final String parameterContextId, final String assetName, final InputStream contents) throws IOException { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + return assetManager.createAsset(parameterContextId, assetName, contents); + } + } + + @Override + public Optional getAsset(final String id) { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + return assetManager.getAsset(id); + } + } + + @Override + public List getAssets(final String parameterContextId) { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + return assetManager.getAssets(parameterContextId); + } + } + + @Override + public Asset createMissingAsset(final String parameterContextId, final String assetName) { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + return assetManager.createMissingAsset(parameterContextId, assetName); + } + } + + @Override + public Optional deleteAsset(final String id) { + try (final NarCloseable narCloseable = NarCloseable.withComponentNarLoader(assetManagerClassLoader)) { + return assetManager.deleteAsset(id); + } + } + }; + + instrumented.initialize(initializationContext); + + return instrumented; + } catch (final Exception e) { + throw new RuntimeException("Failed to create Asset Manager", e); + } + } + + public AssetManager getAssetManager() { + return assetManager; + } + private ProvenanceRepository createProvenanceRepository(final NiFiProperties properties) { final String implementationClassName = properties.getProperty(NiFiProperties.PROVENANCE_REPO_IMPLEMENTATION_CLASS, DEFAULT_PROVENANCE_REPO_IMPLEMENTATION); if (StringUtils.isBlank(implementationClassName)) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java index d36bb469e633..8f260a88b120 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/StandardFlowService.java @@ -17,6 +17,7 @@ package org.apache.nifi.controller; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.asset.AssetSynchronizer; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.AuthorizerCapabilityDetection; import org.apache.nifi.authorization.ManagedAuthorizer; @@ -123,6 +124,7 @@ public class StandardFlowService implements FlowService, ProtocolHandler { private final ClusterCoordinator clusterCoordinator; private final RevisionManager revisionManager; private final NarManager narManager; + private final AssetSynchronizer assetSynchronizer; private volatile SaveReportingTask saveReportingTask; /** @@ -153,9 +155,10 @@ public static StandardFlowService createStandaloneInstance( final NiFiProperties nifiProperties, final RevisionManager revisionManager, final NarManager narManager, + final AssetSynchronizer assetSynchronizer, final Authorizer authorizer) throws IOException { - return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, narManager, authorizer); + return new StandardFlowService(controller, nifiProperties, null, false, null, revisionManager, narManager, assetSynchronizer, authorizer); } public static StandardFlowService createClusteredInstance( @@ -165,9 +168,10 @@ public static StandardFlowService createClusteredInstance( final ClusterCoordinator coordinator, final RevisionManager revisionManager, final NarManager narManager, + final AssetSynchronizer assetSynchronizer, final Authorizer authorizer) throws IOException { - return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, narManager, authorizer); + return new StandardFlowService(controller, nifiProperties, senderListener, true, coordinator, revisionManager, narManager, assetSynchronizer, authorizer); } private StandardFlowService( @@ -178,6 +182,7 @@ private StandardFlowService( final ClusterCoordinator clusterCoordinator, final RevisionManager revisionManager, final NarManager narManager, + final AssetSynchronizer assetSynchronizer, final Authorizer authorizer) throws IOException { this.nifiProperties = nifiProperties; @@ -193,6 +198,7 @@ private StandardFlowService( } this.revisionManager = revisionManager; this.narManager = narManager; + this.assetSynchronizer = assetSynchronizer; this.authorizer = authorizer; if (configuredForClustering) { @@ -962,6 +968,9 @@ private void loadFromConnectionResponse(final ConnectionResponse response) throw // load new controller state loadFromBytes(dataFlow, true, BundleUpdateStrategy.USE_SPECIFIED_OR_COMPATIBLE_OR_GHOST); + // sync assets after loading the flow so that parameter contexts exist first + assetSynchronizer.synchronize(); + // set node ID on controller before we start heartbeating because heartbeat needs node ID clusterCoordinator.setLocalNodeIdentifier(nodeId); clusterCoordinator.setConnected(true); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java index ba12acc106ab..9809a389c3da 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardFlowManager.java @@ -275,7 +275,8 @@ public ProcessGroup createProcessGroup(final String id) { final ProcessGroup group = new StandardProcessGroup(requireNonNull(id), flowController.getControllerServiceProvider(), processScheduler, flowController.getEncryptor(), flowController.getExtensionManager(), flowController.getStateManagerProvider(), this, - flowController.getReloadComponent(), flowController, nifiProperties, statelessGroupNodeFactory); + flowController.getReloadComponent(), flowController, nifiProperties, statelessGroupNodeFactory, + flowController.getAssetManager()); onProcessGroupAdded(group); return group; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardStatelessGroupNodeFactory.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardStatelessGroupNodeFactory.java index 50f4a9b262a3..6c4919e10a3e 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardStatelessGroupNodeFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/flow/StandardStatelessGroupNodeFactory.java @@ -140,6 +140,7 @@ public StatelessGroupNode createStatelessGroupNode(final ProcessGroup group) { .mapSensitiveConfiguration(true) .sensitiveValueEncryptor(value -> value) // No need to encrypt, since we won't be persisting the flow .stateLookup(VersionedComponentStateLookup.IDENTITY_LOOKUP) + .mapAssetReferences(true) .build(); final StatelessGroupFactory statelessGroupFactory = new StatelessGroupFactory() { @@ -234,6 +235,7 @@ public Future> fetch(final Set bundleCoordinates, .counterRepository(flowController.getCounterRepository()) .encryptor(flowController.getEncryptor()) .extensionManager(flowController.getExtensionManager()) + .assetManager(flowController.getAssetManager()) .extensionRepository(extensionRepository) .flowFileEventRepository(flowFileEventRepository) .processScheduler(statelessScheduler) diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java index bd5f5244217f..d3b6e0b6f962 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/AffectedComponentSet.java @@ -355,7 +355,8 @@ public void addAffectedComponents(final FlowDifference difference) { return; } - if (differenceType == DifferenceType.PARAMETER_VALUE_CHANGED || differenceType == DifferenceType.PARAMETER_DESCRIPTION_CHANGED || differenceType == DifferenceType.PARAMETER_REMOVED) { + if (differenceType == DifferenceType.PARAMETER_VALUE_CHANGED || differenceType == DifferenceType.PARAMETER_ASSET_REFERENCES_CHANGED + || differenceType == DifferenceType.PARAMETER_DESCRIPTION_CHANGED || differenceType == DifferenceType.PARAMETER_REMOVED) { addComponentsForParameterUpdate(difference); return; } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java index e719ac5aa62f..dc74d4f17b1b 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedDataflowMapper.java @@ -71,6 +71,7 @@ public VersionedDataflowMapper(final FlowController flowController, final Extens .mapInstanceIdentifiers(true) .mapControllerServiceReferencesToVersionedId(false) .mapFlowRegistryClientId(true) + .mapAssetReferences(true) .build(); flowMapper = new NiFiRegistryFlowMapper(extensionManager, mappingOptions); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java index fee9285ab6ba..c0e97327dcab 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/serialization/VersionedFlowSynchronizer.java @@ -18,6 +18,8 @@ package org.apache.nifi.controller.serialization; import org.apache.commons.collections4.CollectionUtils; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.AuthorizerCapabilityDetection; import org.apache.nifi.authorization.ManagedAuthorizer; @@ -51,6 +53,7 @@ import org.apache.nifi.flow.Bundle; import org.apache.nifi.flow.ExecutionEngine; import org.apache.nifi.flow.ScheduledState; +import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedControllerService; import org.apache.nifi.flow.VersionedExternalFlow; @@ -110,6 +113,7 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -120,6 +124,7 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.GZIPInputStream; import static org.apache.nifi.controller.serialization.FlowSynchronizationUtils.createBundleCoordinate; @@ -446,6 +451,7 @@ private void synchronizeFlow(final FlowController controller, final DataFlow exi .mapInstanceIdentifiers(true) .mapControllerServiceReferencesToVersionedId(false) .mapFlowRegistryClientId(true) + .mapAssetReferences(true) .build(); rootGroup.synchronizeFlow(versionedExternalFlow, syncOptions, flowMappingOptions); @@ -779,7 +785,7 @@ private void inheritParameterContexts(final FlowController controller, final Ver parameterContexts.forEach(context -> namedParameterContexts.put(context.getName(), context)); for (final VersionedParameterContext versionedParameterContext : parameterContexts) { - inheritParameterContext(versionedParameterContext, controller.getFlowManager(), namedParameterContexts, controller.getEncryptor()); + inheritParameterContext(versionedParameterContext, controller.getFlowManager(), namedParameterContexts, controller.getEncryptor(), controller.getAssetManager()); } }); } @@ -788,14 +794,15 @@ private void inheritParameterContext( final VersionedParameterContext versionedParameterContext, final FlowManager flowManager, final Map namedParameterContexts, - final PropertyEncryptor encryptor + final PropertyEncryptor encryptor, + final AssetManager assetManager ) { final ParameterContextManager contextManager = flowManager.getParameterContextManager(); final ParameterContext existingContext = contextManager.getParameterContextNameMapping().get(versionedParameterContext.getName()); if (existingContext == null) { - addParameterContext(versionedParameterContext, flowManager, namedParameterContexts, encryptor); + addParameterContext(versionedParameterContext, flowManager, namedParameterContexts, encryptor, assetManager); } else { - updateParameterContext(versionedParameterContext, existingContext, flowManager, namedParameterContexts, encryptor); + updateParameterContext(versionedParameterContext, existingContext, flowManager, namedParameterContexts, encryptor, assetManager); } } @@ -803,9 +810,10 @@ private void addParameterContext( final VersionedParameterContext versionedParameterContext, final FlowManager flowManager, final Map namedParameterContexts, - final PropertyEncryptor encryptor + final PropertyEncryptor encryptor, + final AssetManager assetManager ) { - final Map parameters = createParameterMap(flowManager, versionedParameterContext, encryptor); + final Map parameters = createParameterMap(flowManager, versionedParameterContext, encryptor, assetManager); final ParameterContextManager contextManager = flowManager.getParameterContextManager(); final List referenceIds = findReferencedParameterContextIds(versionedParameterContext, contextManager, namedParameterContexts); @@ -853,18 +861,13 @@ private List findReferencedParameterContextIds( private Map createParameterMap( final FlowManager flowManager, final VersionedParameterContext versionedParameterContext, - final PropertyEncryptor encryptor + final PropertyEncryptor encryptor, + final AssetManager assetManager ) { final Map providedParameters = getProvidedParameters(flowManager, versionedParameterContext); final Map parameters = new HashMap<>(); for (final VersionedParameter versioned : versionedParameterContext.getParameters()) { - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() - .description(versioned.getDescription()) - .name(versioned.getName()) - .sensitive(versioned.isSensitive()) - .build(); - final String parameterValue; final String rawValue = versioned.getValue(); if (rawValue == null) { @@ -884,36 +887,73 @@ private Map createParameterMap( parameterValue = rawValue; } - final Parameter parameter = new Parameter(descriptor, parameterValue, null, versioned.isProvided()); + final List referencedAssets = getReferencedAssets(versioned, assetManager, versionedParameterContext.getInstanceIdentifier()); + final Parameter parameter = new Parameter.Builder() + .parameterContextId(null) + .name(versioned.getName()) + .description(versioned.getDescription()) + .sensitive(versioned.isSensitive()) + .value(referencedAssets.isEmpty() ? parameterValue : null) + .referencedAssets(referencedAssets) + .provided(versioned.isProvided()) + .build(); + parameters.put(versioned.getName(), parameter); } return parameters; } + private List getReferencedAssets(final VersionedParameter versionedParameter, final AssetManager assetManager, final String contextId) { + final List versionedAssets = versionedParameter.getReferencedAssets(); + if (versionedAssets == null || versionedAssets.isEmpty()) { + return List.of(); + } + + final List assets = new ArrayList<>(); + for (final VersionedAsset versionedAsset : versionedAssets) { + final Optional assetOption = assetManager.getAsset(versionedAsset.getIdentifier()); + if (assetOption.isPresent()) { + assets.add(assetOption.get()); + } else { + assets.add(assetManager.createMissingAsset(contextId, versionedAsset.getName())); + } + } + + return assets; + } + private void updateParameterContext( final VersionedParameterContext versionedParameterContext, final ParameterContext parameterContext, final FlowManager flowManager, final Map namedParameterContexts, - final PropertyEncryptor encryptor + final PropertyEncryptor encryptor, + final AssetManager assetManager ) { - final Map parameters = createParameterMap(flowManager, versionedParameterContext, encryptor); + final Map parameters = createParameterMap(flowManager, versionedParameterContext, encryptor, assetManager); final Map currentValues = new HashMap<>(); - parameterContext.getParameters().values().forEach(param -> currentValues.put(param.getDescriptor().getName(), param.getValue())); + final Map> currentAssetReferences = new HashMap<>(); + parameterContext.getParameters().values().forEach(param -> { + currentValues.put(param.getDescriptor().getName(), param.getValue()); + currentAssetReferences.put(param.getDescriptor().getName(), getAssetIds(param)); + }); final Map updatedParameters = new HashMap<>(); final Set proposedParameterNames = new HashSet<>(); for (final VersionedParameter parameter : versionedParameterContext.getParameters()) { final String parameterName = parameter.getName(); final String currentValue = currentValues.get(parameterName); + final Set currentAssetIds = currentAssetReferences.get(parameterName); - proposedParameterNames.add(parameterName); - if (!Objects.equals(currentValue, parameter.getValue())) { - final Parameter updatedParameterObject = parameters.get(parameterName); + final Parameter updatedParameterObject = parameters.get(parameterName); + final String updatedValue = updatedParameterObject.getValue(); + final Set updatedAssetIds = getAssetIds(updatedParameterObject); + if (!Objects.equals(currentValue, updatedValue) || !currentAssetIds.equals(updatedAssetIds)) { updatedParameters.put(parameterName, updatedParameterObject); } + proposedParameterNames.add(parameterName); } // If any parameters are removed, need to add a null value to the map in order to make sure that the parameter is removed. @@ -939,6 +979,13 @@ private void updateParameterContext( parameterContext.setInheritedParameterContexts(referencedContexts); } + private Set getAssetIds(final Parameter parameter) { + return Stream.ofNullable(parameter.getReferencedAssets()) + .flatMap(Collection::stream) + .map(Asset::getIdentifier) + .collect(Collectors.toSet()); + } + private void inheritControllerServices(final FlowController controller, final VersionedDataflow dataflow, final AffectedComponentSet affectedComponentSet) { final FlowManager flowManager = controller.getFlowManager(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/framework/configuration/FlowControllerConfiguration.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/framework/configuration/FlowControllerConfiguration.java index 2b54066191d3..abc8509a8f70 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/framework/configuration/FlowControllerConfiguration.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/framework/configuration/FlowControllerConfiguration.java @@ -17,6 +17,11 @@ package org.apache.nifi.framework.configuration; import org.apache.nifi.admin.service.AuditService; +import org.apache.nifi.asset.AssetComponentManager; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.asset.AssetSynchronizer; +import org.apache.nifi.asset.StandardAssetComponentManager; +import org.apache.nifi.asset.StandardAssetSynchronizer; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.coordination.heartbeat.HeartbeatMonitor; @@ -248,6 +253,7 @@ public FlowService flowService(@Autowired final NarManager narManager) throws Ex properties, revisionManager, narManager, + assetSynchronizer(), authorizer ); } else { @@ -258,6 +264,7 @@ public FlowService flowService(@Autowired final NarManager narManager) throws Ex clusterCoordinator, revisionManager, narManager, + assetSynchronizer(), authorizer ); } @@ -434,4 +441,34 @@ public NarManager narManager(@Autowired final NarPersistenceProvider narPersiste properties ); } + + /** + * Asset Manager from Flow Controller + * + * @return Asset Manager + */ + @Bean + public AssetManager assetManager() throws Exception { + return flowController().getAssetManager(); + } + + /** + * Asset Synchronizer depends on ClusterCoordinator, WebClientService, and NiFiProperties + * + * @return Asset Synchronizer + */ + @Bean + public AssetSynchronizer assetSynchronizer() throws Exception { + return new StandardAssetSynchronizer(flowController(), clusterCoordinator, webClientService(), properties); + } + + /** + * Affected Component Manager depends on FlowController + * + * @return Affected Component Manager + */ + @Bean + public AssetComponentManager affectedComponentManager() throws Exception { + return new StandardAssetComponentManager(flowController()); + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiClient.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiClient.java index 9f0f4693c51c..e82bd6875100 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiClient.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/NarRestApiClient.java @@ -17,55 +17,35 @@ package org.apache.nifi.nar; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.module.jakarta.xmlbind.JakartaXmlBindAnnotationIntrospector; -import org.apache.commons.io.IOUtils; +import org.apache.nifi.client.NiFiRestApiClient; +import org.apache.nifi.client.NiFiRestApiRetryableException; import org.apache.nifi.web.api.entity.NarSummariesEntity; import org.apache.nifi.web.client.StandardHttpUriBuilder; import org.apache.nifi.web.client.api.HttpRequestBodySpec; import org.apache.nifi.web.client.api.HttpResponseEntity; -import org.apache.nifi.web.client.api.HttpResponseStatus; import org.apache.nifi.web.client.api.WebClientService; import org.apache.nifi.web.client.api.WebClientServiceException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.util.Objects; /** * Encapsulate API calls for listing and downloading NARs through the REST API of a NiFi node. */ -public class NarRestApiClient { +public class NarRestApiClient extends NiFiRestApiClient { private static final Logger LOGGER = LoggerFactory.getLogger(NarRestApiClient.class); - private static final String HTTP_SCHEME = "http"; - private static final String HTTPS_SCHEME = "https"; - private static final String NIFI_API_PATH = "nifi-api"; private static final String CONTROLLER_PATH = "controller"; private static final String NAR_MANAGER_PATH = "nar-manager"; private static final String NARS_PATH = "nars"; private static final String NAR_CONTENT_PATH = "content"; - private final URI baseUri; - private final WebClientService webClientService; - private final ObjectMapper objectMapper; - public NarRestApiClient(final WebClientService webClientService, final String host, final int port, final boolean secure) { - try { - this.baseUri = new URI(secure ? HTTPS_SCHEME : HTTP_SCHEME, null, host, port, null, null, null); - } catch (URISyntaxException e) { - throw new RuntimeException(e); - } - this.webClientService = Objects.requireNonNull(webClientService, "WebClientService is required"); - this.objectMapper = new ObjectMapper(); - this.objectMapper.setAnnotationIntrospector(new JakartaXmlBindAnnotationIntrospector(objectMapper.getTypeFactory())); + super(webClientService, host, port, secure); } public NarSummariesEntity listNarSummaries() { @@ -84,15 +64,10 @@ public NarSummariesEntity listNarSummaries() { // can happen when no nodes are considered connected and result in a 500 exception that can't easily be differentiated from other unknown errors final HttpRequestBodySpec requestBodySpec = webClientService.get() .uri(requestUri) - .header("Accept", "application/json") - .header("X-Request-Replicated", "true"); + .header(ACCEPT_HEADER, APPLICATION_JSON) + .header(X_REQUEST_REPLICATED_HEADER, "true"); - try (final HttpResponseEntity response = requestBodySpec.retrieve()) { - final InputStream responseBody = getResponseBody(requestUri, response); - return objectMapper.readValue(responseBody, NarSummariesEntity.class); - } catch (final WebClientServiceException | IOException e) { - throw new NarRestApiRetryableException(e.getMessage(), e); - } + return executeEntityRequest(requestUri, requestBodySpec, NarSummariesEntity.class); } public InputStream downloadNar(final String identifier) { @@ -112,30 +87,12 @@ public InputStream downloadNar(final String identifier) { try { final HttpResponseEntity response = webClientService.get() .uri(requestUri) - .header("Accept", "application/octet-stream") + .header(ACCEPT_HEADER, APPLICATION_OCTET_STREAM) .retrieve(); return getResponseBody(requestUri, response); } catch (final WebClientServiceException e) { - throw new NarRestApiRetryableException(e.getMessage(), e); + throw new NiFiRestApiRetryableException(e.getMessage(), e); } } - private InputStream getResponseBody(final URI requestUri, final HttpResponseEntity response) { - final int statusCode = response.statusCode(); - if (HttpResponseStatus.OK.getCode() == statusCode) { - return response.body(); - } else { - final String responseMessage; - try { - responseMessage = IOUtils.toString(response.body(), StandardCharsets.UTF_8); - } catch (IOException e) { - throw new NarRestApiRetryableException("Error reading response from %s - %s".formatted(requestUri, statusCode), e); - } - if (statusCode == HttpResponseStatus.CONFLICT.getCode()) { - throw new NarRestApiRetryableException("Error calling %s - %s - %s".formatted(requestUri, statusCode, responseMessage)); - } else { - throw new IllegalStateException("Error calling %s - %s - %s".formatted(requestUri, statusCode, responseMessage)); - } - } - } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java index 95ff21895eac..83ec0f9c0910 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/nar/StandardNarManager.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; +import org.apache.nifi.client.NiFiRestApiRetryableException; import org.apache.nifi.cluster.coordination.ClusterCoordinator; import org.apache.nifi.cluster.protocol.NodeIdentifier; import org.apache.nifi.controller.FlowController; @@ -315,7 +316,7 @@ private NarSummariesEntity getNarSummariesFromCoordinator(final NarRestApiClient private NarSummariesEntity listNarSummaries(final NarRestApiClient narRestApiClient) { try { return narRestApiClient.listNarSummaries(); - } catch (final NarRestApiRetryableException e) { + } catch (final NiFiRestApiRetryableException e) { final Throwable rootCause = ExceptionUtils.getRootCause(e); logger.warn("{}, root cause [{}]: retrying", e.getMessage(), rootCause.getMessage()); return null; diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/parameter/mock/PlaceholderParameterProvider.java b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/parameter/mock/PlaceholderParameterProvider.java index 43f9429c44ce..c59f2bf2fa59 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/parameter/mock/PlaceholderParameterProvider.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/java/org/apache/nifi/parameter/mock/PlaceholderParameterProvider.java @@ -21,7 +21,6 @@ import org.apache.nifi.controller.ControllerService; import org.apache.nifi.parameter.AbstractParameterProvider; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.ParameterProvider; @@ -51,12 +50,13 @@ protected List getSupportedPropertyDescriptors() { @Override public List fetchParameters(final ConfigurationContext context) { final List parameters = Arrays.stream(STATIC_PARAMETERS) - .map(parameterName -> { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder() - .name(parameterName) - .build(); - return new Parameter(parameterDescriptor, parameterName + "-value", null, true); - }) + .map(parameterName -> + new Parameter.Builder() + .name(parameterName) + .value(parameterName + "-value") + .provided(true) + .build() + ) .collect(Collectors.toList()); return Collections.singletonList(new ParameterGroup("Group", parameters)); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties index b41410f31533..dbfa70989795 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/test/resources/flowcontrollertest.nifi.properties @@ -127,4 +127,7 @@ nifi.cluster.manager.protocol.threads=10 nifi.cluster.manager.safemode.duration=0 sec # analytics properties # -nifi.analytics.predict.interval=3 mins \ No newline at end of file +nifi.analytics.predict.interval=3 mins + +# Asset Manager # +nifi.asset.manager.properties.directory=target/assets \ No newline at end of file diff --git a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java index 53623a5619a3..ba208c5a61ba 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-framework-nar-utils/src/main/java/org/apache/nifi/nar/StandardExtensionDiscoveringManager.java @@ -17,6 +17,7 @@ package org.apache.nifi.nar; import org.apache.nifi.annotation.behavior.RequiresInstanceClassLoading; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.authentication.LoginIdentityProvider; import org.apache.nifi.authorization.AccessPolicyProvider; import org.apache.nifi.authorization.Authorizer; @@ -131,6 +132,7 @@ public StandardExtensionDiscoveringManager(final Collection()); definitionMap.put(PythonBridge.class, new HashSet<>()); definitionMap.put(NarPersistenceProvider.class, new HashSet<>()); + definitionMap.put(AssetManager.class, new HashSet<>()); additionalExtensionTypes.forEach(type -> definitionMap.putIfAbsent(type, new HashSet<>())); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java b/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java index 66320bcbc00c..d6b7b7a838fe 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java +++ b/nifi-framework-bundle/nifi-framework/nifi-headless-server/src/main/java/org/apache/nifi/headless/HeadlessNiFiServer.java @@ -159,6 +159,7 @@ public void preDestruction() throws AuthorizerDestructionException { props, null, // revision manager null, // NAR Manager + null, // Asset Synchronizer authorizer); diagnosticsFactory = new BootstrapDiagnosticsFactory(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml index 355917855389..d04fb4478b0c 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/pom.xml @@ -121,6 +121,10 @@ org.apache.nifi.nar.StandardNarPersistenceProvider ./nar_repository + + org.apache.nifi.asset.StandardAssetManager + ./assets + diff --git a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties index f1ef06025198..a018664ab78d 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties +++ b/nifi-framework-bundle/nifi-framework/nifi-resources/src/main/resources/conf/nifi.properties @@ -135,6 +135,10 @@ nifi.status.repository.questdb.persist.location=${nifi.status.repository.questdb nifi.nar.persistence.provider.implementation=${nifi.nar.persistence.provider.implementation} nifi.nar.persistence.provider.properties.directory=${nifi.nar.persistence.provider.properties.directory} +# Asset Management +nifi.asset.manager.implementation=${nifi.asset.manager.implementation} +nifi.asset.manager.properties.directory=${nifi.asset.manager.properties.directory} + # Site to Site properties nifi.remote.input.host= nifi.remote.input.secure=${nifi.remote.input.secure} diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java index 23d97929af51..2d92e5cafb1f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/NiFiServiceFacade.java @@ -92,6 +92,7 @@ import org.apache.nifi.web.api.entity.ActionEntity; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; import org.apache.nifi.web.api.entity.ConfigurationAnalysisEntity; @@ -2910,4 +2911,24 @@ ControllerServiceReferencingComponentsEntity updateControllerServiceReferencingC */ NarSummaryEntity deleteNar(String identifier) throws IOException; + // ---------------------------------------- + // Asset Manager methods + // ---------------------------------------- + + /** + * Verifies the given asset can be deleted from the given parameter context. + * + * @param parameterContextId the parameter context id + * @param assetId the asset id + */ + void verifyDeleteAsset(String parameterContextId, String assetId); + + /** + * Deletes the given asset from the given parameter context. + * + * @param parameterContextId the parameter context id + * @param assetId the asset id + */ + AssetEntity deleteAsset(String parameterContextId, String assetId); + } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java index afeeb61925ec..ed4725e09cd3 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/StandardNiFiServiceFacade.java @@ -26,6 +26,8 @@ import org.apache.nifi.action.Operation; import org.apache.nifi.action.details.FlowChangePurgeDetails; import org.apache.nifi.admin.service.AuditService; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.attribute.expression.language.Query; import org.apache.nifi.authorization.AccessDeniedException; import org.apache.nifi.authorization.AccessPolicy; @@ -204,6 +206,7 @@ import org.apache.nifi.web.api.dto.AccessPolicyDTO; import org.apache.nifi.web.api.dto.AccessPolicySummaryDTO; import org.apache.nifi.web.api.dto.AffectedComponentDTO; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.BulletinBoardDTO; import org.apache.nifi.web.api.dto.BulletinDTO; import org.apache.nifi.web.api.dto.BulletinQueryDTO; @@ -302,6 +305,7 @@ import org.apache.nifi.web.api.entity.ActionEntity; import org.apache.nifi.web.api.entity.ActivateControllerServicesEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ComponentReferenceEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; @@ -486,6 +490,7 @@ public class StandardNiFiServiceFacade implements NiFiServiceFacade { private RuleViolationsManager ruleViolationsManager; private PredictionBasedParallelProcessingService parallelProcessingService; private NarManager narManager; + private AssetManager assetManager; // ----------------------------------------- // Synchronization methods @@ -1400,18 +1405,47 @@ private ComponentValidationResultEntity validateComponent(final ComponentNode co } private Parameter createParameter(final ParameterDTO dto) { - if (dto.getDescription() == null && dto.getSensitive() == null && dto.getValue() == null) { + if (dto.getDescription() == null && dto.getSensitive() == null && dto.getValue() == null && dto.getReferencedAssets() == null) { return null; // null description, sensitivity flag, and value indicates a deletion, which we want to represent as a null Parameter. } - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() + final String dtoValue = dto.getValue(); + final List referencedAssets = dto.getReferencedAssets(); + final boolean referencesAsset = referencedAssets != null && !referencedAssets.isEmpty(); + final String parameterContextId = dto.getParameterContext() == null ? null : dto.getParameterContext().getId(); + + final String value; + List assets = null; + if (dtoValue == null && !referencesAsset && Boolean.TRUE.equals(dto.getValueRemoved())) { + value = null; + } else if (referencesAsset) { + assets = getAssets(referencedAssets); + value = null; + } else { + value = dto.getValue(); + } + + return new Parameter.Builder() .name(dto.getName()) .description(dto.getDescription()) .sensitive(Boolean.TRUE.equals(dto.getSensitive())) + .value(value) + .referencedAssets(assets) + .parameterContextId(parameterContextId) + .provided(dto.getProvided()) .build(); + } - final String parameterContextId = dto.getParameterContext() == null ? null : dto.getParameterContext().getId(); - return new Parameter(descriptor, dto.getValue(), parameterContextId, dto.getProvided()); + private List getAssets(final List referencedAssets) { + return Stream.ofNullable(referencedAssets) + .flatMap(Collection::stream) + .map(AssetReferenceDTO::getId) + .map(this::getAsset) + .collect(Collectors.toList()); + } + + private Asset getAsset(final String assetId) { + return assetManager.getAsset(assetId).orElseThrow(() -> new ResourceNotFoundException("Unable to find asset with id " + assetId)); } @Override @@ -6715,6 +6749,38 @@ public NarSummaryEntity deleteNar(final String identifier) throws IOException { return entityFactory.createNarSummaryEntity(narSummaryDTO); } + @Override + public void verifyDeleteAsset(final String parameterContextId, final String assetId) { + final ParameterContext parameterContext = parameterContextDAO.getParameterContext(parameterContextId); + final Set referencingParameterNames = getReferencingParameterNames(parameterContext, assetId); + if (!referencingParameterNames.isEmpty()) { + final String joinedParametersNames = String.join(", ", referencingParameterNames); + throw new IllegalStateException("Unable to delete Asset [%s] because it is currently references by Parameters [%s]".formatted(assetId, joinedParametersNames)); + } + } + + private Set getReferencingParameterNames(final ParameterContext parameterContext, final String assetId) { + final Set referencingParameterNames = new HashSet<>(); + for (final Parameter parameter : parameterContext.getParameters().values()) { + if (parameter.getReferencedAssets() != null) { + for (final Asset asset : parameter.getReferencedAssets()) { + if (asset.getIdentifier().equals(assetId)) { + referencingParameterNames.add(parameter.getDescriptor().getName()); + } + } + } + } + return referencingParameterNames; + } + + @Override + public AssetEntity deleteAsset(final String parameterContextId, final String assetId) { + verifyDeleteAsset(parameterContextId, assetId); + final Asset deletedAsset = assetManager.deleteAsset(assetId) + .orElseThrow(() -> new ResourceNotFoundException("Asset does not exist with id [%s]".formatted(assetId))); + return dtoFactory.createAssetEntity(deletedAsset); + } + private PermissionsDTO createPermissionDto( final String id, final org.apache.nifi.flow.ComponentType subjectComponentType, @@ -7005,4 +7071,9 @@ public void setParallelProcessingService(PredictionBasedParallelProcessingServic public void setNarManager(final NarManager narManager) { this.narManager = narManager; } + + @Autowired + public void setAssetManager(final AssetManager assetManager) { + this.assetManager = assetManager; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java index 6f22cd7454ea..b06b7aa83db6 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ApplicationResource.java @@ -46,6 +46,8 @@ import org.apache.nifi.remote.exception.NotAuthorizedException; import org.apache.nifi.remote.protocol.ResponseCode; import org.apache.nifi.remote.protocol.http.HttpHeaders; +import org.apache.nifi.security.cert.PeerIdentityProvider; +import org.apache.nifi.security.cert.StandardPeerIdentityProvider; import org.apache.nifi.util.ComponentIdGenerator; import org.apache.nifi.util.NiFiProperties; import org.apache.nifi.web.NiFiServiceFacade; @@ -80,9 +82,12 @@ import jakarta.ws.rs.core.UriBuilder; import jakarta.ws.rs.core.UriInfo; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.core.context.SecurityContextHolder; +import javax.net.ssl.SSLPeerUnverifiedException; import java.net.URI; import java.nio.charset.StandardCharsets; +import java.security.cert.X509Certificate; import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; @@ -95,6 +100,7 @@ import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; import static jakarta.ws.rs.core.Response.Status.NOT_FOUND; import static org.apache.commons.lang3.StringUtils.isEmpty; @@ -124,7 +130,8 @@ public abstract class ApplicationResource { @Context protected UriInfo uriInfo; - protected ApplicationCookieService applicationCookieService = new StandardApplicationCookieService(); + protected final PeerIdentityProvider peerIdentityProvider = new StandardPeerIdentityProvider(); + protected final ApplicationCookieService applicationCookieService = new StandardApplicationCookieService(); protected NiFiProperties properties; private RequestReplicator requestReplicator; private ClusterCoordinator clusterCoordinator; @@ -1301,4 +1308,52 @@ protected void stripNonUiRelevantFields(final ControllerServiceReferencingCompon referencingEntities.forEach(this::stripNonUiRelevantFields); } } + + /** + * @return true if the credentials of the current request contain a certificate that matches an identity of a known cluster node, false otherwise + */ + protected boolean isRequestFromClusterNode() { + final ClusterCoordinator clusterCoordinator = getClusterCoordinator(); + if (clusterCoordinator == null) { + logger.debug("Clustering is not configured"); + return false; + } + + final X509Certificate[] certificates = getAuthenticationCertificates(); + if (certificates == null) { + logger.debug("Client credentials do not contain certificates"); + return false; + } + + final Set clientIdentities; + try { + clientIdentities = peerIdentityProvider.getIdentities(certificates); + } catch (final SSLPeerUnverifiedException e) { + throw new RuntimeException("Unable to get identities from client certificates", e); + } + + final Set nodeIds = getClusterCoordinator().getNodeIdentifiers().stream() + .map(NodeIdentifier::getApiAddress) + .collect(Collectors.toSet()); + + logger.debug("Checking client identities [{}] against cluster node identities [{}]", clientIdentities, nodeIds); + + for (final String clientIdentity : clientIdentities) { + if (nodeIds.contains(clientIdentity)) { + logger.debug("Client identity [{}] is in the list of cluster nodes", clientIdentity); + return true; + } + } + + logger.debug("None of the client identities [{}] are in the list of cluster nodes", clientIdentities); + return false; + } + + private X509Certificate[] getAuthenticationCertificates() { + final Object credentials = SecurityContextHolder.getContext().getAuthentication().getCredentials(); + if (credentials instanceof X509Certificate[]) { + return (X509Certificate[]) credentials; + } + return null; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterContextResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterContextResource.java index 3cf4ab79c13d..d1a557506d1f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterContextResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterContextResource.java @@ -16,22 +16,6 @@ */ package org.apache.nifi.web.api; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -45,6 +29,7 @@ import jakarta.ws.rs.DELETE; import jakarta.ws.rs.DefaultValue; import jakarta.ws.rs.GET; +import jakarta.ws.rs.HeaderParam; import jakarta.ws.rs.HttpMethod; import jakarta.ws.rs.POST; import jakarta.ws.rs.PUT; @@ -52,9 +37,14 @@ import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.HttpHeaders; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.StreamingOutput; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.asset.AssetComponentManager; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.authorization.AuthorizableLookup; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.ComponentAuthorizable; @@ -62,10 +52,18 @@ import org.apache.nifi.authorization.resource.Authorizable; import org.apache.nifi.authorization.user.NiFiUser; import org.apache.nifi.authorization.user.NiFiUserUtils; +import org.apache.nifi.cluster.coordination.ClusterCoordinator; +import org.apache.nifi.cluster.coordination.http.replication.UploadRequest; +import org.apache.nifi.cluster.coordination.http.replication.UploadRequestReplicator; +import org.apache.nifi.cluster.coordination.node.NodeConnectionState; +import org.apache.nifi.cluster.protocol.NodeIdentifier; +import org.apache.nifi.controller.ComponentNode; import org.apache.nifi.controller.ControllerService; -import org.apache.nifi.controller.service.StandardControllerServiceNode; import org.apache.nifi.parameter.ParameterContext; import org.apache.nifi.parameter.ParameterReferencedControllerServiceData; +import org.apache.nifi.processor.DataUnit; +import org.apache.nifi.stream.io.MaxLengthInputStream; +import org.apache.nifi.util.file.FileUtils; import org.apache.nifi.web.NiFiServiceFacade; import org.apache.nifi.web.ResourceNotFoundException; import org.apache.nifi.web.ResumeFlowException; @@ -76,6 +74,8 @@ import org.apache.nifi.web.api.concurrent.StandardAsynchronousWebRequest; import org.apache.nifi.web.api.concurrent.StandardUpdateStep; import org.apache.nifi.web.api.concurrent.UpdateStep; +import org.apache.nifi.web.api.dto.AssetDTO; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.DtoFactory; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterContextUpdateRequestDTO; @@ -84,6 +84,8 @@ import org.apache.nifi.web.api.dto.ParameterDTO; import org.apache.nifi.web.api.dto.RevisionDTO; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultEntity; import org.apache.nifi.web.api.entity.ComponentValidationResultsEntity; import org.apache.nifi.web.api.entity.Entity; @@ -94,6 +96,7 @@ import org.apache.nifi.web.api.entity.ProcessGroupEntity; import org.apache.nifi.web.api.request.ClientIdParameter; import org.apache.nifi.web.api.request.LongParameter; +import org.apache.nifi.web.client.api.HttpResponseStatus; import org.apache.nifi.web.util.ComponentLifecycle; import org.apache.nifi.web.util.ParameterUpdateManager; import org.slf4j.Logger; @@ -102,24 +105,50 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Controller; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + @Controller @Path("/parameter-contexts") @Tag(name = "ParameterContexts") public class ParameterContextResource extends AbstractParameterResource { private static final Logger logger = LoggerFactory.getLogger(ParameterContextResource.class); private static final Pattern VALID_PARAMETER_NAME_PATTERN = Pattern.compile("[A-Za-z0-9 ._\\-]+"); + private static final String FILENAME_HEADER = "Filename"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private static final String UPLOAD_CONTENT_TYPE = "application/octet-stream"; + private static final long MAX_ASSET_SIZE_BYTES = (long) DataUnit.GB.toB(1); private NiFiServiceFacade serviceFacade; private Authorizer authorizer; private DtoFactory dtoFactory; private ComponentLifecycle clusterComponentLifecycle; private ComponentLifecycle localComponentLifecycle; + private AssetManager assetManager; + private AssetComponentManager assetComponentManager; + private UploadRequestReplicator uploadRequestReplicator; private ParameterUpdateManager parameterUpdateManager; - private RequestManager, List> updateRequestManager - = new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Parameter Context Update Thread"); - private RequestManager validationRequestManager = new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), - "Parameter Context Validation Thread"); + private final RequestManager, List> updateRequestManager = + new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Parameter Context Update Thread"); + private final RequestManager validationRequestManager = + new AsyncRequestManager<>(100, TimeUnit.MINUTES.toMillis(1L), "Parameter Context Validation Thread"); @PostConstruct public void init() { @@ -131,6 +160,15 @@ private void authorizeReadParameterContext(final String parameterContextId) { throw new IllegalArgumentException("Parameter Context ID must be specified"); } + // In order for a node to sync its assets with the cluster coordinator, it needs to be able to READ from any parameter context, and parameter contexts can have specific policies + // which would require users adding the node identities to all of these policies, so this identifies if the incoming request is made directly by a known node identity and allows + // it to bypass standard authorization, meaning a node is automatically granted READ to any parameter context + final NiFiUser currentUser = NiFiUserUtils.getNiFiUser(); + if (isRequestFromClusterNode()) { + logger.debug("Authorizing READ on ParameterContext[{}] to cluster node [{}]", parameterContextId, currentUser.getIdentity()); + return; + } + serviceFacade.authorizeAccess(lookup -> { final Authorizable parameterContext = lookup.getParameterContext(parameterContextId); parameterContext.authorize(authorizer, RequestAction.READ, NiFiUserUtils.getNiFiUser()); @@ -181,7 +219,6 @@ public Response getParameterContext( return generateOkResponse(entity).build(); } - @POST @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @@ -257,8 +294,7 @@ public Response createParameterContext( summary = "Modifies a Parameter Context", responses = @ApiResponse(content = @Content(schema = @Schema(implementation = ParameterContextEntity.class))), description = "This endpoint will update a Parameter Context to match the provided entity. However, this request will fail if any component is running and is referencing a Parameter in " + - "the " + - "Parameter Context. Generally, this endpoint is not called directly. Instead, an update request should be submitted by making a POST to the " + + "the Parameter Context. Generally, this endpoint is not called directly. Instead, an update request should be submitted by making a POST to the " + "/parameter-contexts/update-requests endpoint. That endpoint will, in turn, call this endpoint.", security = { @SecurityRequirement(name = "Read - /parameter-contexts/{id}"), @@ -323,6 +359,293 @@ public Response updateParameterContext( ); } + @POST + @Consumes(MediaType.APPLICATION_OCTET_STREAM) + @Produces(MediaType.APPLICATION_JSON) + @Path("{contextId}/assets") + @Operation( + summary = "Creates a new Asset in the given Parameter Context", + responses = @ApiResponse(content = @Content(schema = @Schema(implementation = AssetEntity.class))), + description = "This endpoint will create a new Asset in the given Parameter Context. The Asset will be created with the given name and the contents of the file that is uploaded. " + + "The Asset will be created in the given Parameter Context, and will be available for use by any component that references the Parameter Context.", + security = { + @SecurityRequirement(name = "Read - /parameter-contexts/{parameterContextId}"), + @SecurityRequirement(name = "Write - /parameter-contexts/{parameterContextId}"), + @SecurityRequirement(name = "Read - for every component that is affected by the update"), + @SecurityRequirement(name = "Write - for every component that is affected by the update"), + @SecurityRequirement(name = "Read - for every currently inherited parameter context") + } + ) + @ApiResponses( + value = { + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response createAsset( + @PathParam("contextId") final String contextId, + @HeaderParam(FILENAME_HEADER) final String assetName, + @Parameter(description = "The contents of the asset.", required = true) final InputStream assetContents) throws IOException { + + // Validate input + if (StringUtils.isBlank(assetName)) { + throw new IllegalArgumentException(FILENAME_HEADER + " header is required"); + } + if (assetContents == null) { + throw new IllegalArgumentException("Asset contents must be specified."); + } + + final String sanitizedAssetName = FileUtils.getSanitizedFilename(assetName); + if (!assetName.equals(sanitizedAssetName)) { + throw new IllegalArgumentException(FILENAME_HEADER + " header contains an invalid file name"); + } + + // If clustered and not all nodes are connected, do not allow creating an asset. + // Generally, we allow the flow to be modified when nodes are disconnected, but we do not allow creating an asset because + // the cluster has no mechanism for synchronizing those assets after the upload. + final ClusterCoordinator clusterCoordinator = getClusterCoordinator(); + if (clusterCoordinator != null) { + final Set disconnectedNodes = clusterCoordinator.getNodeIdentifiers(NodeConnectionState.CONNECTING, NodeConnectionState.DISCONNECTED, NodeConnectionState.DISCONNECTING); + if (!disconnectedNodes.isEmpty()) { + throw new IllegalStateException("Cannot create an Asset because the following %s nodes are not currently connected: %s".formatted(disconnectedNodes.size(), disconnectedNodes)); + } + } + + // Get the context or throw ResourceNotFoundException + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final ParameterContextEntity contextEntity = serviceFacade.getParameterContext(contextId, false, user); + final Set affectedComponents = serviceFacade.getComponentsAffectedByParameterContextUpdate(Collections.singletonList(contextEntity.getComponent())); + final Optional previousAsset = assetManager.getAssets(contextId).stream().filter(asset -> asset.getName().equals(sanitizedAssetName)).findAny(); + + // Authorize the request + serviceFacade.authorizeAccess(lookup -> { + // Verify READ and WRITE permissions for user, for the Parameter Context itself + final ParameterContext parameterContext = lookup.getParameterContext(contextId); + parameterContext.authorize(authorizer, RequestAction.READ, user); + parameterContext.authorize(authorizer, RequestAction.WRITE, user); + + // Verify READ and WRITE permissions for user, for every component that is affected + // This is necessary because this end-point may be called to replace the content of an asset that is referenced in a parameter that is already in use + if (previousAsset.isPresent()) { + affectedComponents.forEach(component -> parameterUpdateManager.authorizeAffectedComponent(component, lookup, user, true, true)); + } + }); + + // If we need to replicate the request, we do so using the Upload Request Replicator, rather than the typical replicate() method. + // This is because Upload Request Replication works differently in that it needs to be able to replicate the InputStream multiple times, + // so it must create a file on disk to do so and then use the file's content to replicate the request. It also bypasses the two-phase + // commit process that is used for other requests because doing so would result in uploading the file twice to each node or providing a + // different request for each of the two phases. + + final long startTime = System.currentTimeMillis(); + final InputStream maxLengthInputStream = new MaxLengthInputStream(assetContents, MAX_ASSET_SIZE_BYTES); + + final AssetEntity assetEntity; + if (isReplicateRequest()) { + final UploadRequest uploadRequest = new UploadRequest.Builder() + .user(NiFiUserUtils.getNiFiUser()) + .filename(sanitizedAssetName) + .identifier(UUID.randomUUID().toString()) + .contents(maxLengthInputStream) + .header(FILENAME_HEADER, sanitizedAssetName) + .header(CONTENT_TYPE_HEADER, UPLOAD_CONTENT_TYPE) + .exampleRequestUri(getAbsolutePath()) + .responseClass(AssetEntity.class) + .successfulResponseStatus(HttpResponseStatus.OK.getCode()) + .build(); + assetEntity = uploadRequestReplicator.upload(uploadRequest); + } else { + final String existingContextId = contextEntity.getId(); + final Asset asset = assetManager.createAsset(existingContextId, sanitizedAssetName, maxLengthInputStream); + assetEntity = dtoFactory.createAssetEntity(asset); + previousAsset.ifPresent(value -> assetComponentManager.restartReferencingComponentsAsync(value)); + } + + final AssetDTO assetDTO = assetEntity.getAsset(); + final long elapsedTime = System.currentTimeMillis() - startTime; + logger.info("Creation of asset [id={},name={}] in parameter context [{}] completed in {} ms", assetDTO.getId(), assetDTO.getName(), contextId, elapsedTime); + + return generateOkResponse(assetEntity).build(); + } + + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{contextId}/assets") + @Operation( + summary = "Lists the assets that belong to the Parameter Context with the given ID", + responses = @ApiResponse(content = @Content(schema = @Schema(implementation = AssetsEntity.class))), + description = "Lists the assets that belong to the Parameter Context with the given ID.", + security = { + @SecurityRequirement(name = "Read - /parameter-contexts/{id}") + } + ) + @ApiResponses( + value = { + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response getAssets( + @Parameter(description = "The ID of the Parameter Context") + @PathParam("contextId") final String parameterContextId + ) { + // authorize access + authorizeReadParameterContext(parameterContextId); + + if (isReplicateRequest()) { + return replicate(HttpMethod.GET); + } + + // Get the assets from the manager rather than from the context's values since there could be assets that were created, but never referenced + final List paramContextAssets = assetManager.getAssets(parameterContextId); + + final AssetsEntity assetsEntity = new AssetsEntity(); + assetsEntity.setAssets(paramContextAssets.stream() + .map(dtoFactory::createAssetEntity) + .toList()); + + return generateOkResponse(assetsEntity).build(); + } + + @GET + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @Path("{contextId}/assets/{assetId}") + @Operation( + summary = "Retrieves the content of the asset with the given id", + responses = @ApiResponse(content = @Content(schema = @Schema(implementation = byte[].class))), + security = { + @SecurityRequirement(name = "Read - /parameter-contexts/{id}") + } + ) + @ApiResponses( + value = { + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response getAssetContent( + @Parameter(description = "The ID of the Parameter Context") + @PathParam("contextId") + final String parameterContextId, + @Parameter(description = "The ID of the Asset") + @PathParam("assetId") + final String assetId + ) { + // authorize access + authorizeReadParameterContext(parameterContextId); + + final Asset asset = assetManager.getAsset(assetId) + .orElseThrow(() -> new ResourceNotFoundException("Asset does not exist with the id %s".formatted(assetId))); + + if (!asset.getParameterContextIdentifier().equals(parameterContextId)) { + throw new ResourceNotFoundException("Asset does not exist with id %s".formatted(assetId)); + } + + if (!asset.getFile().exists()) { + throw new IllegalStateException("Content does not exist for asset with id %s".formatted(assetId)); + } + + final StreamingOutput streamingOutput = (outputStream) -> { + try (final InputStream assetInputStream = new FileInputStream(asset.getFile())) { + assetInputStream.transferTo(outputStream); + } + }; + + return generateOkResponse(streamingOutput) + .header(HttpHeaders.CONTENT_DISPOSITION, String.format("attachment; filename=\"%s\"", asset.getName())) + .build(); + } + + @DELETE + @Consumes(MediaType.WILDCARD) + @Produces(MediaType.APPLICATION_JSON) + @Path("{contextId}/assets/{assetId}") + @Operation( + summary = "Deletes an Asset from the given Parameter Context", + responses = @ApiResponse(content = @Content(schema = @Schema(implementation = AssetEntity.class))), + description = "This endpoint will create a new Asset in the given Parameter Context. The Asset will be created with the given name and the contents of the file that is uploaded. " + + "The Asset will be created in the given Parameter Context, and will be available for use by any component that references the Parameter Context.", + security = { + @SecurityRequirement(name = "Read - /parameter-contexts/{parameterContextId}"), + @SecurityRequirement(name = "Write - /parameter-contexts/{parameterContextId}"), + @SecurityRequirement(name = "Read - for every component that is affected by the update"), + @SecurityRequirement(name = "Write - for every component that is affected by the update"), + @SecurityRequirement(name = "Read - for every currently inherited parameter context") + } + ) + @ApiResponses( + value = { + @ApiResponse(responseCode = "400", description = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), + @ApiResponse(responseCode = "401", description = "Client could not be authenticated."), + @ApiResponse(responseCode = "403", description = "Client is not authorized to make this request."), + @ApiResponse(responseCode = "404", description = "The specified resource could not be found."), + @ApiResponse(responseCode = "409", description = "The request was valid but NiFi was not in the appropriate state to process it.") + } + ) + public Response deleteAsset( + @QueryParam(DISCONNECTED_NODE_ACKNOWLEDGED) @DefaultValue("false") + final Boolean disconnectedNodeAcknowledged, + @Parameter(description = "The ID of the Parameter Context") + @PathParam("contextId") + final String parameterContextId, + @Parameter(description = "The ID of the Asset") + @PathParam("assetId") + final String assetId + ) { + if (StringUtils.isBlank(parameterContextId)) { + throw new IllegalArgumentException("Parameter context id is required"); + } + if (StringUtils.isBlank(assetId)) { + throw new IllegalArgumentException("Asset id is required"); + } + + if (isReplicateRequest()) { + return replicate(HttpMethod.DELETE); + } else if (isDisconnectedFromCluster()) { + verifyDisconnectedNodeModification(disconnectedNodeAcknowledged); + } + + // Get the context or throw ResourceNotFoundException + final NiFiUser user = NiFiUserUtils.getNiFiUser(); + final ParameterContextEntity contextEntity = serviceFacade.getParameterContext(parameterContextId, false, user); + + final AssetDTO assetDTO = new AssetDTO(); + assetDTO.setId(assetId); + + final AssetEntity assetEntity = new AssetEntity(); + assetEntity.setAsset(assetDTO); + + // Need to call into service facade with a lock to ensure that the parameter context can't be updated to + // reference the asset being deleted at the same time that we are verifying no parameters reference it + return withWriteLock( + serviceFacade, + assetEntity, + lookup -> { + // Deletion of an asset will only be allowed when it is not referenced by any parameters, so we only need to + // authorize that the user has access to modify the context which is READ and WRITE on the context itself + final ParameterContext parameterContext = lookup.getParameterContext(contextEntity.getId()); + parameterContext.authorize(authorizer, RequestAction.READ, user); + parameterContext.authorize(authorizer, RequestAction.WRITE, user); + }, + () -> serviceFacade.verifyDeleteAsset(contextEntity.getId(), assetId), + requestEntity -> { + final String requestAssetId = requestEntity.getAsset().getId(); + final AssetEntity deletedAsset = serviceFacade.deleteAsset(contextEntity.getId(), requestAssetId); + return generateOkResponse(deletedAsset).build(); + } + ); + } @POST @Consumes(MediaType.APPLICATION_JSON) @@ -382,6 +705,7 @@ public Response submitParameterContextUpdate( } validateParameterNames(contextDto); + validateAssetReferences(contextDto); // We will perform the updating of the Parameter Context in a background thread because it can be a long-running process. // In order to do this, we will need some objects that are only available as Thread-Local variables to the current @@ -422,58 +746,7 @@ public Response submitParameterContextUpdate( // Verify READ and WRITE permissions for user, for every component that is affected affectedComponents.forEach(component -> parameterUpdateManager.authorizeAffectedComponent(component, lookup, user, true, true)); - Set parametersEntities = requestEntity.getComponent().getParameters(); - for (ParameterEntity parameterEntity : parametersEntities) { - String parameterName = parameterEntity.getParameter().getName(); - List referencedControllerServiceDataSet = parameterContext - .getParameterReferenceManager() - .getReferencedControllerServiceData(parameterContext, parameterName); - - Set> referencedControllerServiceTypes = referencedControllerServiceDataSet - .stream() - .map(ParameterReferencedControllerServiceData::getReferencedControllerServiceType) - .collect(Collectors.toSet()); - - if (referencedControllerServiceTypes.size() > 1) { - throw new IllegalStateException("Parameter is used by multiple different types of controller service references"); - } else if (!referencedControllerServiceTypes.isEmpty()) { - Optional parameterOptional = parameterContext.getParameter(parameterName); - if (parameterOptional.isPresent()) { - String currentParameterValue = parameterOptional.get().getValue(); - if (currentParameterValue != null) { - ComponentAuthorizable currentControllerService = lookup.getControllerService(currentParameterValue); - if (currentControllerService != null) { - Authorizable currentControllerServiceAuthorizable = currentControllerService.getAuthorizable(); - if (currentControllerServiceAuthorizable != null) { - currentControllerServiceAuthorizable.authorize(authorizer, RequestAction.READ, user); - currentControllerServiceAuthorizable.authorize(authorizer, RequestAction.WRITE, user); - } - } - } - } - - String newParameterValue = parameterEntity.getParameter().getValue(); - if (newParameterValue != null) { - ComponentAuthorizable newControllerService = lookup.getControllerService(newParameterValue); - if (newControllerService != null) { - Authorizable newControllerServiceAuthorizable = newControllerService.getAuthorizable(); - if (newControllerServiceAuthorizable != null) { - newControllerServiceAuthorizable.authorize(authorizer, RequestAction.READ, user); - newControllerServiceAuthorizable.authorize(authorizer, RequestAction.WRITE, user); - - if ( - !referencedControllerServiceTypes.iterator().next() - .isAssignableFrom( - ((StandardControllerServiceNode) newControllerServiceAuthorizable).getComponent().getClass() - ) - ) { - throw new IllegalArgumentException("New Parameter value attempts to reference an incompatible controller service"); - } - } - } - } - } - } + validateControllerServiceReferences(requestEntity, lookup, parameterContext, user); }, () -> { // Verify Request @@ -483,6 +756,91 @@ public Response submitParameterContextUpdate( ); } + private void validateControllerServiceReferences(final ParameterContextEntity requestEntity, final AuthorizableLookup lookup, final ParameterContext parameterContext, final NiFiUser user) { + final Set parametersEntities = requestEntity.getComponent().getParameters(); + for (ParameterEntity parameterEntity : parametersEntities) { + final String parameterName = parameterEntity.getParameter().getName(); + final List referencedControllerServiceDataSet = + parameterContext.getParameterReferenceManager().getReferencedControllerServiceData(parameterContext, parameterName); + + final Set> referencedControllerServiceTypes = referencedControllerServiceDataSet.stream() + .map(ParameterReferencedControllerServiceData::getReferencedControllerServiceType) + .collect(Collectors.toSet()); + + if (referencedControllerServiceTypes.size() > 1) { + throw new IllegalStateException("Parameter is used by multiple different types of controller service references"); + } else if (!referencedControllerServiceTypes.isEmpty()) { + final Optional parameterOptional = parameterContext.getParameter(parameterName); + if (parameterOptional.isPresent()) { + final String currentParameterValue = parameterOptional.get().getValue(); + if (currentParameterValue != null) { + final ComponentAuthorizable currentControllerService = lookup.getControllerService(currentParameterValue); + if (currentControllerService != null) { + final Authorizable currentControllerServiceAuthorizable = currentControllerService.getAuthorizable(); + currentControllerServiceAuthorizable.authorize(authorizer, RequestAction.READ, user); + currentControllerServiceAuthorizable.authorize(authorizer, RequestAction.WRITE, user); + } + } + } + + final String newParameterValue = parameterEntity.getParameter().getValue(); + if (newParameterValue != null) { + final ComponentAuthorizable newControllerService = lookup.getControllerService(newParameterValue); + if (newControllerService != null) { + final Authorizable newControllerServiceAuthorizable = newControllerService.getAuthorizable(); + if (newControllerServiceAuthorizable != null) { + newControllerServiceAuthorizable.authorize(authorizer, RequestAction.READ, user); + newControllerServiceAuthorizable.authorize(authorizer, RequestAction.WRITE, user); + + final Class firstClass = referencedControllerServiceTypes.iterator().next(); + final ComponentNode componentNode = (ComponentNode) newControllerServiceAuthorizable; + final Class componentClass = componentNode.getComponent().getClass(); + + if (!firstClass.isAssignableFrom(componentClass)) { + throw new IllegalArgumentException("New Parameter value attempts to reference an incompatible controller service"); + } + } + } + } + } + } + } + + + private void validateAssetReferences(final ParameterContextDTO parameterContextDto) { + if (parameterContextDto.getParameters() != null) { + for (final ParameterEntity entity : parameterContextDto.getParameters()) { + final List referencedAssets = entity.getParameter().getReferencedAssets(); + if (referencedAssets == null) { + continue; + } + + for (final AssetReferenceDTO referencedAsset : referencedAssets) { + if (StringUtils.isBlank(referencedAsset.getId())) { + throw new IllegalArgumentException("Asset reference id cannot be blank"); + } + final Asset asset = assetManager.getAsset(referencedAsset.getId()) + .orElseThrow(() -> new IllegalArgumentException("Request contains a reference to an Asset (%s) that does not exist".formatted(referencedAsset))); + if (!asset.getParameterContextIdentifier().equals(parameterContextDto.getId())) { + throw new IllegalArgumentException("Request contains a reference to an Asset (%s) that does not exist in Parameter Context (%s)" + .formatted(referencedAsset, parameterContextDto.getId())); + } + } + + if (!referencedAssets.isEmpty() && entity.getParameter().getValue() != null) { + throw new IllegalArgumentException( + "Request contains a Parameter (%s) with both a value and a reference to an Asset. A Parameter may have either a value or a reference to an Asset, but not both." + .formatted(entity.getParameter().getName())); + } + + if (!referencedAssets.isEmpty() && entity.getParameter().getSensitive() != null && entity.getParameter().getSensitive()) { + throw new IllegalArgumentException("Request contains a sensitive Parameter (%s) with references to an Assets. Sensitive parameters may not reference Assets." + .formatted(entity.getParameter().getName())); + } + } + } + } + private void validateParameterNames(final ParameterContextDTO parameterContextDto) { if (parameterContextDto.getParameters() != null) { for (final ParameterEntity entity : parameterContextDto.getParameters()) { @@ -873,10 +1231,10 @@ private Response submitUpdateRequest(final Revision requestRevision, final Initi // Submit the request to be performed in the background final Consumer, List>> updateTask = asyncRequest -> { try { - final List updatedParameterContextEntities = parameterUpdateManager - .updateParameterContexts(asyncRequest, requestWrapper.getComponentLifecycle(), requestWrapper.getExampleUri(), - requestWrapper.getReferencingComponents(), requestWrapper.isReplicateRequest(), requestRevision, - Collections.singletonList(requestWrapper.getParameterContextEntity())); + final List updatedParameterContextEntities = parameterUpdateManager.updateParameterContexts( + asyncRequest, requestWrapper.getComponentLifecycle(), requestWrapper.getExampleUri(), + requestWrapper.getReferencingComponents(), requestWrapper.isReplicateRequest(), requestRevision, + List.of(requestWrapper.getParameterContextEntity())); asyncRequest.markStepComplete(updatedParameterContextEntities); } catch (final ResumeFlowException rfe) { @@ -1117,4 +1475,18 @@ public void setDtoFactory(final DtoFactory dtoFactory) { this.dtoFactory = dtoFactory; } + @Autowired + public void setAssetManager(final AssetManager assetManager) { + this.assetManager = assetManager; + } + + @Autowired(required = false) + public void setUploadRequestReplicator(final UploadRequestReplicator uploadRequestReplicator) { + this.uploadRequestReplicator = uploadRequestReplicator; + } + + @Autowired + public void setAffectedComponentManager(final AssetComponentManager assetComponentManager) { + this.assetComponentManager = assetComponentManager; + } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterProviderResource.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterProviderResource.java index 44bec487e0fe..8447cbd9596f 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterProviderResource.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/ParameterProviderResource.java @@ -17,24 +17,6 @@ package org.apache.nifi.web.api; import com.google.common.base.Functions; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Collectors; - import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Content; @@ -130,6 +112,23 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Controller; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.stream.Collectors; + /** * RESTful endpoint for managing a Parameter Provider. */ @@ -1694,7 +1693,7 @@ public NiFiUser getUser() { } @Autowired - public void setServiceFacade(NiFiServiceFacade serviceFacade) { + public void setServiceFacade(final NiFiServiceFacade serviceFacade) { this.serviceFacade = serviceFacade; } @@ -1720,7 +1719,7 @@ public DtoFactory getDtoFactory() { } @Autowired - public void setDtoFactory(DtoFactory dtoFactory) { + public void setDtoFactory(final DtoFactory dtoFactory) { this.dtoFactory = dtoFactory; } } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java index 27daac467f53..b7e0e5072031 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/api/dto/DtoFactory.java @@ -40,6 +40,7 @@ import org.apache.nifi.annotation.documentation.CapabilityDescription; import org.apache.nifi.annotation.documentation.DeprecationNotice; import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.asset.Asset; import org.apache.nifi.authorization.AccessPolicy; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.AuthorizerCapabilityDetection; @@ -232,6 +233,7 @@ import org.apache.nifi.web.api.entity.AccessPolicySummaryEntity; import org.apache.nifi.web.api.entity.AffectedComponentEntity; import org.apache.nifi.web.api.entity.AllowableValueEntity; +import org.apache.nifi.web.api.entity.AssetEntity; import org.apache.nifi.web.api.entity.BulletinEntity; import org.apache.nifi.web.api.entity.ComponentReferenceEntity; import org.apache.nifi.web.api.entity.ConnectionStatusSnapshotEntity; @@ -256,6 +258,7 @@ import jakarta.ws.rs.WebApplicationException; +import java.io.File; import java.text.Collator; import java.text.NumberFormat; import java.util.ArrayList; @@ -1509,6 +1512,26 @@ private ParameterProviderConfigurationEntity createParameterProviderConfiguratio return config; } + public AssetEntity createAssetEntity(final Asset asset) { + final AssetEntity entity = new AssetEntity(); + entity.setAsset(createAssetDto(asset)); + return entity; + } + + public AssetDTO createAssetDto(final Asset asset) { + final File assetFile = asset.getFile(); + final AssetDTO dto = new AssetDTO(); + dto.setId(asset.getIdentifier()); + dto.setName(asset.getName()); + dto.setDigest(asset.getDigest().orElse(null)); + dto.setMissingContent(!assetFile.exists()); + return dto; + } + + public AssetReferenceDTO createAssetReferenceDto(final Asset asset) { + return new AssetReferenceDTO(asset.getIdentifier(), asset.getName()); + } + public ParameterEntity createParameterEntity(final ParameterContext parameterContext, final Parameter parameter, final RevisionManager revisionManager, final ParameterContextLookup parameterContextLookup) { final ParameterDTO dto = createParameterDto(parameterContext, parameter, revisionManager, parameterContextLookup); @@ -1533,6 +1556,8 @@ public ParameterDTO createParameterDto(final ParameterContext parameterContext, dto.setValue(descriptor.isSensitive() ? SENSITIVE_VALUE_MASK : parameter.getValue()); } dto.setProvided(parameter.isProvided()); + final List assets = parameter.getReferencedAssets(); + dto.setReferencedAssets(assets == null ? List.of() : parameter.getReferencedAssets().stream().map(this::createAssetReferenceDto).toList()); final ParameterReferenceManager parameterReferenceManager = parameterContext.getParameterReferenceManager(); diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardParameterContextDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardParameterContextDAO.java index a54332d5b748..b151f0857507 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardParameterContextDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/dao/impl/StandardParameterContextDAO.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.web.dao.impl; +import org.apache.nifi.asset.Asset; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.RequestAction; import org.apache.nifi.authorization.user.NiFiUser; @@ -29,11 +31,11 @@ import org.apache.nifi.groups.ProcessGroup; import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.parameter.ParameterProviderConfiguration; import org.apache.nifi.parameter.StandardParameterProviderConfiguration; import org.apache.nifi.web.ResourceNotFoundException; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.ParameterContextDTO; import org.apache.nifi.web.api.dto.ParameterContextReferenceDTO; import org.apache.nifi.web.api.dto.ParameterDTO; @@ -42,10 +44,13 @@ import org.apache.nifi.web.api.entity.ParameterEntity; import org.apache.nifi.web.api.entity.ParameterProviderConfigurationEntity; import org.apache.nifi.web.dao.ParameterContextDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -54,10 +59,14 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import java.util.stream.Stream; @Repository public class StandardParameterContextDAO implements ParameterContextDAO { + private static final Logger logger = LoggerFactory.getLogger(StandardParameterContextDAO.class); + private FlowManager flowManager; + private AssetManager assetManager; private Authorizer authorizer; @Override @@ -92,6 +101,7 @@ public ParameterContext createParameterContext(final ParameterContextDTO paramet resolveInheritedParameterContexts(parameterContextDto); verifyParameterSourceConflicts(parameterContextDto); + verifyAssets(parameterContextDto, parameters); final AtomicReference parameterContextReference = new AtomicReference<>(); flowManager.withParameterContextResolution(() -> { @@ -108,6 +118,7 @@ public ParameterContext createParameterContext(final ParameterContextDTO paramet } parameterContextReference.set(parameterContext); + logger.info("Created parameter context with id [{}] and name [{}]", parameterContext.getIdentifier(), parameterContext.getName()); }); return parameterContextReference.get(); } @@ -195,7 +206,7 @@ public Map getParameters(final ParameterContextDTO parameterC throw new IllegalArgumentException("Cannot specify a Parameter without a name"); } - final boolean deletion = parameterDto.getDescription() == null && parameterDto.getSensitive() == null && parameterDto.getValue() == null; + final boolean deletion = parameterDto.getDescription() == null && parameterDto.getSensitive() == null && parameterDto.getValue() == null && parameterDto.getReferencedAssets() == null; if (deletion) { parameterMap.put(parameterDto.getName().trim(), null); } else { @@ -208,16 +219,20 @@ public Map getParameters(final ParameterContextDTO parameterC } private Parameter createParameter(final ParameterDTO dto, final ParameterContext context) { - final ParameterDescriptor descriptor = new ParameterDescriptor.Builder() - .name(dto.getName()) - .description(dto.getDescription()) - .sensitive(Boolean.TRUE.equals(dto.getSensitive())) - .build(); + final String dtoValue = dto.getValue(); + final List referencedAssets = dto.getReferencedAssets(); + final boolean referencesAsset = referencedAssets != null && !referencedAssets.isEmpty(); + final String parameterContextId = dto.getParameterContext() == null ? null : dto.getParameterContext().getId(); final String value; - if (dto.getValue() == null && Boolean.TRUE.equals(dto.getValueRemoved())) { + List assets = null; + if (dtoValue == null && !referencesAsset && Boolean.TRUE.equals(dto.getValueRemoved())) { // Value is being explicitly set to null value = null; + } else if (referencesAsset) { + // Parameter is referencing an asset. The value is not used. + assets = getAssets(referencedAssets); + value = null; } else if (dto.getValue() == null && context != null) { // Value was just never supplied. Use the value from the Parameter Context, if there is one. final Optional optionalParameter = context.getParameter(dto.getName()); @@ -226,9 +241,27 @@ private Parameter createParameter(final ParameterDTO dto, final ParameterContext value = dto.getValue(); } - final String parameterContextId = dto.getParameterContext() == null ? null : dto.getParameterContext().getId(); + return new Parameter.Builder() + .name(dto.getName()) + .description(dto.getDescription()) + .sensitive(Boolean.TRUE.equals(dto.getSensitive())) + .parameterContextId(parameterContextId) + .value(value) + .referencedAssets(assets) + .provided(dto.getProvided()) + .build(); + } + + private List getAssets(final List referencedAssets) { + return Stream.ofNullable(referencedAssets) + .flatMap(Collection::stream) + .map(AssetReferenceDTO::getId) + .map(this::getAsset) + .collect(Collectors.toList()); + } - return new Parameter(descriptor, value, parameterContextId, dto.getProvided()); + private Asset getAsset(final String assetId) { + return assetManager.getAsset(assetId).orElseThrow(() -> new ResourceNotFoundException("Unable to find asset with id " + assetId)); } @Override @@ -270,6 +303,7 @@ public ParameterContext updateParameterContext(final ParameterContextDTO paramet final List inheritedParameterContexts = getInheritedParameterContexts(parameterContextDto); context.setInheritedParameterContexts(inheritedParameterContexts); } + return context; } @@ -281,7 +315,7 @@ public List getInheritedParameterContexts(final ParameterConte if (parameterContextDto.getInheritedParameterContexts() != null) { inheritedParameterContexts.addAll(parameterContextDto.getInheritedParameterContexts().stream() .map(entity -> flowManager.getParameterContextManager().getParameterContext(entity.getComponent().getId())) - .collect(Collectors.toList())); + .toList()); } return inheritedParameterContexts; @@ -306,10 +340,25 @@ public void verifyUpdate(final ParameterContextDTO parameterContextDto, final bo final List inheritedParameterContexts = getInheritedParameterContexts(parameterContextDto); final Map parameters = parameterContextDto.getParameters() == null ? Collections.emptyMap() : getParameters(parameterContextDto, currentContext); + verifyAssets(parameterContextDto, parameters); currentContext.verifyCanUpdateParameterContext(parameters, inheritedParameterContexts); } + public void verifyAssets(final ParameterContextDTO parameterContextDto, final Map parameters) { + for (final Parameter parameter : parameters.values()) { + if (parameter != null) { + final List assets = parameter.getReferencedAssets() == null ? Collections.emptyList() : parameter.getReferencedAssets(); + for (final Asset asset : assets) { + if (!asset.getParameterContextIdentifier().equals(parameterContextDto.getId())) { + throw new IllegalArgumentException(String.format("Parameter [%s] is not allowed to reference asset [%s] which does not belong to parameter context [%s]", + parameter.getDescriptor().getName(), asset.getName(), parameterContextDto.getId())); + } + } + } + } + } + @Override public ParameterProvider getParameterProvider(final ParameterContextDTO parameterContextDTO) { return getParameterProvider(parameterContextDTO.getParameterProviderConfiguration()); @@ -434,11 +483,19 @@ public void deleteParameterContext(final String parameterContextId) { .forEach(referencesToRemove::add); referencesToRemove.forEach(provider::removeReference); }); + + // Remove all assets + final Set assetIds = Optional.ofNullable(assetManager.getAssets(parameterContextId)) + .orElse(Collections.emptyList()).stream() + .map(Asset::getIdentifier) + .collect(Collectors.toSet()); + assetIds.forEach(assetId -> assetManager.deleteAsset(assetId)); } @Autowired public void setFlowController(final FlowController flowController) { this.flowManager = flowController.getFlowManager(); + this.assetManager = flowController.getAssetManager(); } private List getBoundProcessGroups(final String parameterContextId) { diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ParameterUpdateManager.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ParameterUpdateManager.java index 6fae18556587..469274d56d09 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ParameterUpdateManager.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/main/java/org/apache/nifi/web/util/ParameterUpdateManager.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.web.util; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.apache.nifi.authorization.AuthorizableLookup; import org.apache.nifi.authorization.Authorizer; import org.apache.nifi.authorization.RequestAction; @@ -37,8 +39,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/TestStandardParameterContextDAO.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/TestStandardParameterContextDAO.java index 361a68e535a5..4d4cee8bc2ff 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/TestStandardParameterContextDAO.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-api/src/test/java/org/apache/nifi/web/dao/impl/TestStandardParameterContextDAO.java @@ -27,7 +27,6 @@ import org.apache.nifi.controller.flow.FlowManager; import org.apache.nifi.parameter.Parameter; import org.apache.nifi.parameter.ParameterContext; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterReferenceManager; import org.apache.nifi.parameter.StandardParameterContext; import org.apache.nifi.parameter.StandardParameterContextManager; @@ -99,8 +98,7 @@ void setUp() { .name("Inherited") .build(); final Map parameters = new HashMap<>(); - parameters.put("inherited-param", new Parameter(new ParameterDescriptor.Builder().name("inherited-param").build(), - "value", null, true)); + parameters.put("inherited-param", new Parameter.Builder().name("inherited-param").value("value").provided(true).build()); inheritedContext.setParameters(parameters); parameterContextLookup.addParameterContext(inheritedContext); } diff --git a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationProvider.java b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationProvider.java index e3495ef83e7f..88d3af42d552 100644 --- a/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationProvider.java +++ b/nifi-framework-bundle/nifi-framework/nifi-web/nifi-web-security/src/main/java/org/apache/nifi/web/security/x509/X509AuthenticationProvider.java @@ -81,7 +81,7 @@ public Authentication authenticate(Authentication authentication) throws Authent final X509AuthenticationRequestToken request = (X509AuthenticationRequestToken) authentication; // attempt to authenticate if certificates were found - final X509Certificate[] certificates = request.getCertificates();; + final X509Certificate[] certificates = request.getCertificates(); final AuthenticationResponse authenticationResponse; try { authenticationResponse = certificateIdentityProvider.authenticate(certificates); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java index 23d02391c2bd..4d7f91e5fed2 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/DifferenceType.java @@ -175,6 +175,11 @@ public enum DifferenceType { */ PARAMETER_VALUE_CHANGED("Parameter Value Changed"), + /** + * The assets referenced by the Parameter is different in each of the flows + */ + PARAMETER_ASSET_REFERENCES_CHANGED("Parameter Asset References Changed"), + /** * The description of the Parameter is different in each of the flows */ diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java index cc8f4340e1b1..1b87ce1fab6b 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/main/java/org/apache/nifi/registry/flow/diff/StandardFlowComparator.java @@ -19,6 +19,7 @@ import org.apache.nifi.components.PortFunction; import org.apache.nifi.flow.ExecutionEngine; +import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedConnection; import org.apache.nifi.flow.VersionedControllerService; @@ -43,12 +44,14 @@ import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Function; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; public class StandardFlowComparator implements FlowComparator { private static final Logger logger = LoggerFactory.getLogger(StandardFlowComparator.class); @@ -258,6 +261,22 @@ void compare(final VersionedParameterContext contextA, final VersionedParameterC differences.add(new StandardFlowDifference(DifferenceType.PARAMETER_VALUE_CHANGED, contextA, contextB, name, valueA, valueB, description)); } + final Set assetIdsA = Stream.ofNullable(parameterA.getReferencedAssets()) + .flatMap(Collection::stream) + .map(VersionedAsset::getIdentifier) + .collect(Collectors.toSet()); + final Set assetIdsB = Stream.ofNullable(parameterB.getReferencedAssets()) + .flatMap(Collection::stream) + .map(VersionedAsset::getIdentifier) + .collect(Collectors.toSet()); + if (!assetIdsA.equals(assetIdsB)) { + final List valueA = parameterA.getReferencedAssets(); + final List valueB = parameterB.getReferencedAssets(); + final String description = differenceDescriptor.describeDifference(DifferenceType.PARAMETER_ASSET_REFERENCES_CHANGED, + flowA.getName(), flowB.getName(), contextA, contextB, name, valueA, valueB); + differences.add(new StandardFlowDifference(DifferenceType.PARAMETER_ASSET_REFERENCES_CHANGED, contextA, contextB, name, valueA, valueB, description)); + } + if (!Objects.equals(parameterA.getDescription(), parameterB.getDescription())) { final String valueA = parameterA.getDescription(); final String valueB = parameterB.getDescription(); diff --git a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/test/java/org/apache/nifi/registry/flow/diff/TestStandardFlowComparator.java b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/test/java/org/apache/nifi/registry/flow/diff/TestStandardFlowComparator.java index c09715859bac..9bc40497b6cc 100644 --- a/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/test/java/org/apache/nifi/registry/flow/diff/TestStandardFlowComparator.java +++ b/nifi-registry/nifi-registry-core/nifi-registry-flow-diff/src/test/java/org/apache/nifi/registry/flow/diff/TestStandardFlowComparator.java @@ -17,6 +17,7 @@ package org.apache.nifi.registry.flow.diff; +import org.apache.nifi.flow.VersionedAsset; import org.apache.nifi.flow.VersionedComponent; import org.apache.nifi.flow.VersionedParameter; import org.apache.nifi.flow.VersionedParameterContext; @@ -27,12 +28,14 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Function; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertSame; public class TestStandardFlowComparator { @@ -110,11 +113,59 @@ public void testSensitiveParametersDecryptedBeforeCompare() { assertEquals(1, numContainingValue); } + @Test + public void testAssetReferencesChanged() { + final VersionedAsset assetA = createAsset("assetA", "assetA-file.txt"); + final VersionedAsset assetB = createAsset("assetB", "assetB-file.txt"); + final VersionedAsset assetC = createAsset("assetC", "assetC-file.txt"); + + final Set parametersA = new HashSet<>(); + parametersA.add(createParameter("Param 1", null, false, List.of(assetA, assetB, assetC))); + parametersA.add(createParameter("Param 2", "Param 2 Value", false)); + + final Set parametersB = new HashSet<>(); + parametersB.add(createParameter("Param 1", null, false, List.of(assetA, assetC))); + parametersB.add(createParameter("Param 2", "Param 2 Value", false)); + + final VersionedParameterContext contextA = new VersionedParameterContext(); + contextA.setIdentifier("contextA"); + contextA.setInstanceIdentifier("contextAInstanceId"); + contextA.setParameters(parametersA); + + final VersionedParameterContext contextB = new VersionedParameterContext(); + contextB.setIdentifier("contextB"); + contextB.setInstanceIdentifier("contextBInstanceId"); + contextB.setParameters(parametersB); + + final Set differences = new HashSet<>(); + comparator.compare(contextA, contextB, differences); + + assertEquals(1, differences.size()); + + final FlowDifference difference = differences.iterator().next(); + assertNotNull(difference); + assertEquals(DifferenceType.PARAMETER_ASSET_REFERENCES_CHANGED, difference.getDifferenceType()); + assertEquals(contextA.getIdentifier(), difference.getComponentA().getIdentifier()); + assertEquals(contextB.getIdentifier(), difference.getComponentB().getIdentifier()); + } + private VersionedParameter createParameter(final String name, final String value, final boolean sensitive) { + return createParameter(name, value, sensitive, null); + } + + private VersionedParameter createParameter(final String name, final String value, final boolean sensitive, final List referencedAssets) { final VersionedParameter parameter = new VersionedParameter(); parameter.setName(name); parameter.setValue(sensitive ? "enc{" + decryptedToEncrypted.get(value) + "}" : value); parameter.setSensitive(sensitive); + parameter.setReferencedAssets(referencedAssets); return parameter; } + + private VersionedAsset createAsset(final String id, final String name) { + final VersionedAsset asset = new VersionedAsset(); + asset.setIdentifier(id); + asset.setName(name); + return asset; + } } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StandardStatelessEngine.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StandardStatelessEngine.java index 0da271feff2a..92c3457f9199 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StandardStatelessEngine.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StandardStatelessEngine.java @@ -35,6 +35,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.attribute.expression.language.VariableImpact; import org.apache.nifi.bundle.Bundle; import org.apache.nifi.bundle.BundleCoordinate; @@ -109,6 +110,7 @@ public class StandardStatelessEngine implements StatelessEngine { private final StatelessStateManagerProvider stateManagerProvider; private final PropertyEncryptor propertyEncryptor; private final ProcessScheduler processScheduler; + private final AssetManager assetManager; private final KerberosConfig kerberosConfig; private final FlowFileEventRepository flowFileEventRepository; private final ProvenanceRepository provenanceRepository; @@ -139,6 +141,7 @@ private StandardStatelessEngine(final Builder builder) { this.provenanceRepository = requireNonNull(builder.provenanceRepository, "Provenance Repository must be provided"); this.extensionRepository = requireNonNull(builder.extensionRepository, "Extension Repository must be provided"); this.counterRepository = requireNonNull(builder.counterRepository, "Counter Repository must be provided"); + this.assetManager = requireNonNull(builder.assetManager, "Asset Manager must be provided"); this.statusTaskInterval = parseDuration(builder.statusTaskInterval); this.componentEnableTimeout = parseDuration(builder.componentEnableTimeout); @@ -543,7 +546,7 @@ private void overrideParameters(final Map parameterCon final String parameterName = parameter.getDescriptor().getName(); if (parameterValueProvider.isParameterDefined(contextName, parameterName)) { final String providedValue = parameterValueProvider.getParameterValue(contextName, parameterName); - final Parameter updatedParameter = new Parameter(parameter.getDescriptor(), providedValue, parameter.getParameterContextId(), parameter.isProvided()); + final Parameter updatedParameter = new Parameter.Builder().fromParameter(parameter).value(providedValue).build(); updatedParameters.put(parameterName, updatedParameter); } } @@ -575,8 +578,10 @@ private void registerParameterContext(final ParameterContextDefinition parameter } final Parameter existingParameter = optionalParameter.get(); - final Parameter updatedParameter = new Parameter(existingParameter.getDescriptor(), parameterDefinition.getValue(), existingParameter.getParameterContextId(), - existingParameter.isProvided()); + final Parameter updatedParameter = new Parameter.Builder() + .fromParameter(existingParameter) + .value(parameterDefinition.getValue()) + .build(); parameters.put(parameterName, updatedParameter); } } @@ -661,6 +666,11 @@ public Duration getStatusTaskInterval() { return statusTaskInterval; } + @Override + public AssetManager getAssetManager() { + return assetManager; + } + public static class Builder { private ExtensionManager extensionManager = null; private BulletinRepository bulletinRepository = null; @@ -674,6 +684,7 @@ public static class Builder { private CounterRepository counterRepository = null; private String statusTaskInterval = null; private String componentEnableTimeout = null; + private AssetManager assetManager = null; public Builder extensionManager(final ExtensionManager extensionManager) { this.extensionManager = extensionManager; @@ -735,6 +746,11 @@ public Builder componentEnableTimeout(final String componentEnableTimeout) { return this; } + public Builder assetManager(final AssetManager assetManager) { + this.assetManager = assetManager; + return this; + } + public StandardStatelessEngine build() { return new StandardStatelessEngine(this); } diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessEngine.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessEngine.java index 260b64b1b615..d453af193d1c 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessEngine.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessEngine.java @@ -17,6 +17,7 @@ package org.apache.nifi.stateless.engine; +import org.apache.nifi.asset.AssetManager; import org.apache.nifi.components.state.StateManagerProvider; import org.apache.nifi.components.validation.ValidationTrigger; import org.apache.nifi.controller.ProcessScheduler; @@ -71,4 +72,6 @@ public interface StatelessEngine { CounterRepository getCounterRepository(); Duration getStatusTaskInterval(); + + AssetManager getAssetManager(); } \ No newline at end of file diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java index 92e133cca56c..565857006c81 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/engine/StatelessFlowManager.java @@ -230,7 +230,8 @@ public ProcessGroup createProcessGroup(final String id) { statelessEngine.getReloadComponent(), new StatelessNodeTypeProvider(), null, - group -> null); + group -> null, + statelessEngine.getAssetManager()); } @Override diff --git a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java index 757c011e49df..6762d1de5dc1 100644 --- a/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java +++ b/nifi-stateless/nifi-stateless-bundle/nifi-stateless-engine/src/main/java/org/apache/nifi/stateless/flow/StandardStatelessDataflowFactory.java @@ -17,7 +17,12 @@ package org.apache.nifi.stateless.flow; +import org.apache.nifi.asset.AssetManager; +import org.apache.nifi.asset.AssetManagerInitializationContext; +import org.apache.nifi.asset.AssetReferenceLookup; +import org.apache.nifi.asset.StandardAssetManager; import org.apache.nifi.components.state.StatelessStateManagerProvider; +import org.apache.nifi.controller.NodeTypeProvider; import org.apache.nifi.controller.kerberos.KerberosConfig; import org.apache.nifi.controller.repository.ContentRepository; import org.apache.nifi.controller.repository.ContentRepositoryContext; @@ -68,6 +73,7 @@ import org.apache.nifi.stateless.engine.StatelessEngineConfiguration; import org.apache.nifi.stateless.engine.StatelessEngineInitializationContext; import org.apache.nifi.stateless.engine.StatelessFlowManager; +import org.apache.nifi.stateless.engine.StatelessNodeTypeProvider; import org.apache.nifi.stateless.engine.StatelessProcessContextFactory; import org.apache.nifi.stateless.engine.StatelessProvenanceAuthorizableFactory; import org.apache.nifi.stateless.repository.ByteArrayContentRepository; @@ -87,7 +93,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; public class StandardStatelessDataflowFactory implements StatelessDataflowFactory { @@ -123,6 +131,26 @@ public StatelessDataflow createDataflow(final StatelessEngineConfiguration engin engineConfiguration.isLogExtensionDiscovery() ); + final AssetManager assetManager = new StandardAssetManager(); + final File assetDir = new File(workingDir, "assets"); + final AssetManagerInitializationContext assetManagerInitializationContext = new AssetManagerInitializationContext() { + @Override + public AssetReferenceLookup getAssetReferenceLookup() { + return Set::of; + } + + @Override + public Map getProperties() { + return Map.of(StandardAssetManager.ASSET_STORAGE_LOCATION_PROPERTY, assetDir.getAbsolutePath()); + } + + @Override + public NodeTypeProvider getNodeTypeProvider() { + return new StatelessNodeTypeProvider(); + } + }; + assetManager.initialize(assetManagerInitializationContext); + flowFileEventRepo = new RingBufferEventRepository(5); final StatelessStateManagerProvider stateManagerProvider = new StatelessStateManagerProvider(); @@ -188,6 +216,7 @@ private synchronized PropertyEncryptor getEncryptor() { .bulletinRepository(bulletinRepository) .encryptor(lazyInitializedEncryptor) .extensionManager(extensionManager) + .assetManager(assetManager) .stateManagerProvider(stateManagerProvider) .processScheduler(processScheduler) .kerberosConfiguration(kerberosConfig) diff --git a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java index f835f27418dd..ae3b23b4a2cf 100644 --- a/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java +++ b/nifi-system-tests/nifi-system-test-extensions-bundle/nifi-system-test-extensions/src/main/java/org/apache/nifi/parameter/tests/system/PropertiesParameterProvider.java @@ -20,7 +20,6 @@ import org.apache.nifi.controller.ConfigurationContext; import org.apache.nifi.parameter.AbstractParameterProvider; import org.apache.nifi.parameter.Parameter; -import org.apache.nifi.parameter.ParameterDescriptor; import org.apache.nifi.parameter.ParameterGroup; import org.apache.nifi.parameter.ParameterProvider; import org.apache.nifi.processor.util.StandardValidators; @@ -69,12 +68,11 @@ private List fetchParametersFromProperties(final String parametersPro throw new RuntimeException("Could not parse parameters as properties: " + parametersPropertiesValue); } return parameters.entrySet().stream() - .map(entry -> { - final ParameterDescriptor parameterDescriptor = new ParameterDescriptor.Builder() - .name(entry.getKey().toString()) - .build(); - return new Parameter(parameterDescriptor, entry.getValue().toString(), null, true); - }) + .map(entry -> new Parameter.Builder() + .name(entry.getKey().toString()) + .value(entry.getValue().toString()) + .provided(true) + .build()) .collect(Collectors.toList()); } } diff --git a/nifi-system-tests/nifi-system-test-suite/pom.xml b/nifi-system-tests/nifi-system-test-suite/pom.xml index 3904d7dc878f..e1376afd4db2 100644 --- a/nifi-system-tests/nifi-system-test-suite/pom.xml +++ b/nifi-system-tests/nifi-system-test-suite/pom.xml @@ -13,7 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. --> - + nifi-system-tests org.apache.nifi @@ -106,7 +107,7 @@ - org.apache.rat + org.apache.rat apache-rat-plugin @@ -116,6 +117,8 @@ src/test/resources/flows/missing-connection/without-connection.json.gz src/test/resources/keystore.jks src/test/resources/truststore.jks + src/test/resources/sample-assets/helloworld.txt + src/test/resources/sample-assets/helloworld2.txt @@ -383,6 +386,5 @@ 2.0.0-SNAPSHOT nar - diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java index f9aa09a39692..c8dbcbebf522 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiClientUtil.java @@ -31,6 +31,7 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ProcessorClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.VersionsClient; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; import org.apache.nifi.web.api.dto.BundleDTO; import org.apache.nifi.web.api.dto.ConfigVerificationResultDTO; import org.apache.nifi.web.api.dto.ConnectableDTO; @@ -583,8 +584,16 @@ public ParameterEntity createParameterEntity(final String name, final String des return entity; } - public ParameterContextEntity getParameterContext(final String contextId) throws NiFiClientException, IOException { - return nifiClient.getParamContextClient().getParamContext(contextId, false); + public ParameterEntity createAssetReferenceParameterEntity(final String name, final List referencedAssets) { + final ParameterDTO dto = new ParameterDTO(); + dto.setName(name); + dto.setReferencedAssets(referencedAssets.stream().map(assetId -> new AssetReferenceDTO(assetId)).toList()); + dto.setValue(null); + dto.setProvided(false); + + final ParameterEntity entity = new ParameterEntity(); + entity.setParameter(dto); + return entity; } public ParameterContextEntity createParameterContextEntity(final String name, final String description, final Set parameters) { @@ -719,6 +728,26 @@ public ParameterContextUpdateRequestEntity updateParameterContext(final Paramete return nifiClient.getParamContextClient().updateParamContext(entityUpdate); } + public ParameterContextUpdateRequestEntity updateParameterAssetReferences(final ParameterContextEntity existingEntity, final Map> assetReferences) + throws NiFiClientException, IOException { + + final ParameterContextDTO component = existingEntity.getComponent(); + final List inheritedParameterContextIds = component.getInheritedParameterContexts() == null ? null : + component.getInheritedParameterContexts().stream().map(ParameterContextReferenceEntity::getId).collect(Collectors.toList()); + + final Set parameterEntities = new HashSet<>(); + assetReferences.forEach((paramName, references) -> parameterEntities.add(createAssetReferenceParameterEntity(paramName, references))); + existingEntity.getComponent().setParameters(parameterEntities); + + final ParameterContextEntity entityUpdate = createParameterContextEntity(existingEntity.getComponent().getName(), existingEntity.getComponent().getDescription(), + parameterEntities, inheritedParameterContextIds, null); + entityUpdate.setId(existingEntity.getId()); + entityUpdate.setRevision(existingEntity.getRevision()); + entityUpdate.getComponent().setId(existingEntity.getComponent().getId()); + + return nifiClient.getParamContextClient().updateParamContext(entityUpdate); + } + public void waitForParameterContextRequestToComplete(final String contextId, final String requestId) throws NiFiClientException, IOException, InterruptedException { while (true) { final ParameterContextUpdateRequestEntity entity = nifiClient.getParamContextClient().getParamContextUpdateRequest(contextId, requestId); @@ -1336,6 +1365,7 @@ public void deleteAll(final String groupId) throws NiFiClientException, IOExcept for (final ProcessorEntity processorEntity : flowDto.getProcessors()) { processorEntity.setDisconnectedNodeAcknowledged(true); getProcessorClient().deleteProcessor(processorEntity); + logger.info("Deleted processor [{}]", processorEntity.getId()); } // Delete all Controller Services @@ -1367,6 +1397,7 @@ public void deleteAll(final String groupId) throws NiFiClientException, IOExcept for (final ProcessGroupEntity childGroupEntity : flowDto.getProcessGroups()) { childGroupEntity.setDisconnectedNodeAcknowledged(true); deleteAll(childGroupEntity.getId()); + nifiClient.getProcessGroupClient().deleteProcessGroup(childGroupEntity); } } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java index cb6e955d6d96..bb9b88ccfb88 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/NiFiSystemIT.java @@ -154,8 +154,17 @@ public void teardown() throws Exception { logger.info("Beginning teardown"); try { - Exception destroyFlowFailure = null; + // In some cases a test can pass, but still leave a clustered instance with one of the nodes in a bad state, if the instance then gets reused + // it will cause later tests to fail, so it is better to destroy the environment if the cluster is in a bad state at the end of a test + final NiFiInstance nifiInstance = nifiRef.get(); + if (nifiInstance != null && nifiInstance.isClustered() && (!isCoordinatorElected() || !allNodesConnected(nifiInstance.getNumberOfNodes()))) { + logger.info("Clustered environment is in a bad state, will completely tear down the environments and start with a clean environment for the next test."); + instanceCache.poison(nifiInstance); + cleanup(); + return; + } + Exception destroyFlowFailure = null; if (isDestroyFlowAfterEachTest()) { try { destroyFlow(); @@ -598,6 +607,11 @@ protected boolean isCoordinatorElected() throws NiFiClientException, IOException return false; } + protected boolean allNodesConnected(int expectedNodeCount) throws NiFiClientException, IOException { + final ClusterSummaryEntity clusterSummary = getNifiClient().getFlowClient().getClusterSummary(); + return expectedNodeCount == clusterSummary.getClusterSummary().getConnectedNodeCount(); + } + protected void reconnectNode(final int nodeIndex) throws NiFiClientException, IOException { final NodeEntity nodeEntity = getNodeEntity(nodeIndex); nodeEntity.getNode().setStatus(NodeConnectionState.CONNECTING.name()); diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ClusteredParameterContextIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ClusteredParameterContextIT.java index 4f4b1cd9240d..9c41cdf8f532 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ClusteredParameterContextIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ClusteredParameterContextIT.java @@ -17,6 +17,25 @@ package org.apache.nifi.tests.system.parameters; import org.apache.nifi.tests.system.NiFiInstanceFactory; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.util.file.FileUtils; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.ConnectionEntity; +import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ProcessorEntity; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Repeats all tests in ParameterContextIT but in a clustered mode @@ -26,4 +45,182 @@ public class ClusteredParameterContextIT extends ParameterContextIT { public NiFiInstanceFactory getInstanceFactory() { return createTwoNodeInstanceFactory(); } + + @Test + public void testSynchronizeAssets() throws NiFiClientException, IOException, InterruptedException { + waitForAllNodesConnected(); + + final Map paramValues = Map.of("name", "foo", "fileToIngest", ""); + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testSynchronizeAssets", paramValues); + + // Set the Parameter Context on the root Process Group + setParameterContext("root", paramContext); + + // Create a Processor and update it to reference Parameter "name" + final ProcessorEntity ingest = getClientUtil().createProcessor("IngestFile"); + getClientUtil().updateProcessorProperties(ingest, Map.of("Filename", "#{fileToIngest}", "Delete File", "false")); + getClientUtil().updateProcessorSchedulingPeriod(ingest, "10 mins"); + + // Create an asset and update the parameter to reference the asset + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetFile); + + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(ingest, terminate, "success"); + waitForValidProcessor(ingest.getId()); + + getClientUtil().startProcessor(ingest); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + final String contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertEquals("Hello, World!", contents); + + // Check that the file exists in the assets directory. + final File node1Dir = getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1AssetsDir = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1AssetsDir, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2AssetsDir = new File(node2Dir, "assets"); + final File node2ContextDir = new File(node2AssetsDir, paramContext.getId()); + assertTrue(node2ContextDir.exists()); + + // List assets and verify the expected asset is returned + final AssetsEntity assetListing = assertAssetListing(paramContext.getId(), 1); + assertAssetExists(asset, assetListing); + + // Stop node2 + disconnectNode(2); + getNiFiInstance().getNodeInstance(2).stop(); + + // Delete node2's assets + FileUtils.deleteFilesInDir(node2AssetsDir, (dir, name) -> true, null, true, true); + assertTrue(node2AssetsDir.delete()); + assertFalse(node2AssetsDir.exists()); + + // Start node2 again + getNiFiInstance().getNodeInstance(2).start(true); + reconnectNode(2); + waitForAllNodesConnected(); + + // Verify node2 asset directories are back + assertTrue(node2AssetsDir.exists()); + assertTrue(node2ContextDir.exists()); + + final File[] node2AssetFiles = node2ContextDir.listFiles(); + assertNotNull(node2AssetFiles); + assertEquals(1, node2AssetFiles.length); + + // Ensure ingest processor is valid + waitForValidProcessor(ingest.getId()); + + getClientUtil().stopProcessor(ingest); + getClientUtil().waitForStoppedProcessor(ingest.getId()); + } + + @Test + public void testSynchronizeAssetsAfterChangingReferences() throws NiFiClientException, IOException, InterruptedException { + waitForAllNodesConnected(); + + final Map paramValues = Map.of("name", "foo", "filesToIngest", ""); + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testSynchronizeAssetsAfterChangingReferences", paramValues); + + // Set the Parameter Context on the root Process Group + setParameterContext("root", paramContext); + + // Create a Processor and update it to reference Parameter "name" + final ProcessorEntity generateFlowFile = getClientUtil().createProcessor("GenerateFlowFile"); + getClientUtil().updateProcessorProperties(generateFlowFile, Map.of("Text", "#{filesToIngest}")); + getClientUtil().updateProcessorSchedulingPeriod(generateFlowFile, "10 mins"); + + // Create two assets and update the parameter to reference the assets + final AssetEntity asset1 = createAsset(paramContext.getId(), new File("src/test/resources/sample-assets/helloworld.txt")); + final AssetEntity asset2 = createAsset(paramContext.getId(), new File("src/test/resources/sample-assets/helloworld2.txt")); + final List assetIds = List.of(asset1.getAsset().getId(), asset2.getAsset().getId()); + + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences(paramContext, Map.of("filesToIngest", assetIds)); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Connect the Generate processor to a Terminate processor and generate flow files + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(generateFlowFile, terminate, "success"); + waitForValidProcessor(generateFlowFile.getId()); + + getClientUtil().startProcessor(generateFlowFile); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + + // Verify flow files reference both assets + final String flowFileContents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertTrue(flowFileContents.contains(asset1.getAsset().getName())); + assertTrue(flowFileContents.contains(asset2.getAsset().getName())); + + // Verify the contents of the assets directory for each node + final File node1Dir = getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1AssetsDir = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1AssetsDir, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2AssetsDir = new File(node2Dir, "assets"); + final File node2ContextDir = new File(node2AssetsDir, paramContext.getId()); + assertTrue(node2ContextDir.exists()); + + // List assets and verify the expected asset is returned + final AssetsEntity assetListing = assertAssetListing(paramContext.getId(), 2); + assertAssetExists(asset1, assetListing); + + // Stop node2 + disconnectNode(2); + getNiFiInstance().getNodeInstance(2).stop(); + + // Delete node2's assets + FileUtils.deleteFilesInDir(node2AssetsDir, (dir, name) -> true, null, true, true); + assertTrue(node2AssetsDir.delete()); + assertFalse(node2AssetsDir.exists()); + + // Modify the parameter context on node1 to only reference asset1 + final ParameterContextEntity retrievedContext = getNifiClient().getParamContextClient().getParamContext(paramContext.getId(), false); + final ParameterContextUpdateRequestEntity changeAssetsUpdateRequest = getClientUtil().updateParameterAssetReferences( + retrievedContext, Map.of("filesToIngest", List.of(asset1.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), changeAssetsUpdateRequest.getRequest().getRequestId()); + + // Start node2 again + getNiFiInstance().getNodeInstance(2).start(true); + reconnectNode(2); + waitForAllNodesConnected(); + + // Verify node2 asset directories are back + assertTrue(node2AssetsDir.exists()); + assertTrue(node2ContextDir.exists()); + + // Stop generate processor and clear queue + getClientUtil().stopProcessor(generateFlowFile); + getClientUtil().waitForStoppedProcessor(generateFlowFile.getId()); + + getClientUtil().startProcessor(terminate); + waitForQueueCount(connection.getId(), 0); + + getClientUtil().stopProcessor(terminate); + getClientUtil().waitForStoppedProcessor(terminate.getId()); + + // Start generate again + getClientUtil().startProcessor(generateFlowFile); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + + // Verify flow files only reference the first asset + final String flowFile1Contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertTrue(flowFile1Contents.contains(asset1.getAsset().getName())); + assertFalse(flowFile1Contents.contains(asset2.getAsset().getName())); + + final String flowFile2Contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 1); + assertTrue(flowFile2Contents.contains(asset1.getAsset().getName())); + assertFalse(flowFile2Contents.contains(asset2.getAsset().getName())); + + getClientUtil().stopProcessor(generateFlowFile); + getClientUtil().waitForStoppedProcessor(generateFlowFile.getId()); + } } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java index 5fdb353d2e0e..324dd8adbfe7 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java +++ b/nifi-system-tests/nifi-system-test-suite/src/test/java/org/apache/nifi/tests/system/parameters/ParameterContextIT.java @@ -19,6 +19,7 @@ import org.apache.nifi.parameter.ParameterProviderConfiguration; import org.apache.nifi.parameter.ParameterSensitivity; import org.apache.nifi.parameter.StandardParameterProviderConfiguration; +import org.apache.nifi.tests.system.NiFiInstance; import org.apache.nifi.tests.system.NiFiSystemIT; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; @@ -26,6 +27,9 @@ import org.apache.nifi.web.api.dto.ParameterDTO; import org.apache.nifi.web.api.dto.ProcessorConfigDTO; import org.apache.nifi.web.api.entity.AffectedComponentEntity; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; +import org.apache.nifi.web.api.entity.ConnectionEntity; import org.apache.nifi.web.api.entity.ControllerServiceEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; @@ -37,6 +41,8 @@ import org.apache.nifi.web.api.entity.ProcessorEntity; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; @@ -54,12 +60,15 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; public class ParameterContextIT extends NiFiSystemIT { + private static final Logger logger = LoggerFactory.getLogger(ParameterContextIT.class); + @Test public void testCreateParameterContext() throws NiFiClientException, IOException { final Set parameterEntities = new HashSet<>(); @@ -667,6 +676,327 @@ private void testProcessorRestartedWhenParameterChanged(final String propertyVal assertEquals(getNumberOfNodes(), counterValues.get("Scheduled").longValue()); } + @Test + public void testAssetReference() throws NiFiClientException, IOException, InterruptedException { + // Create Parameter Context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testAssetReference", Map.of("name", "foo", "fileToIngest", "")); + + // Set the Parameter Context on the root Process Group + setParameterContext("root", paramContext); + + // Create a Processor and update it to reference a parameter + final ProcessorEntity ingest = getClientUtil().createProcessor("IngestFile"); + getClientUtil().updateProcessorProperties(ingest, Map.of("Filename", "#{fileToIngest}", "Delete File", "false")); + getClientUtil().updateProcessorSchedulingPeriod(ingest, "10 mins"); + + // Create an asset + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetFile); + + // Update parameter to reference the asset + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Connect the ingest processor to terminate processor and produce flow files + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(ingest, terminate, "success"); + waitForValidProcessor(ingest.getId()); + + getClientUtil().startProcessor(ingest); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + final String contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertEquals("Hello, World!", contents); + + // Check that the file exists in the assets directory. + final File node1Dir = getNumberOfNodes() == 1 ? getNiFiInstance().getInstanceDirectory() : getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1Assets = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1Assets, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNumberOfNodes() == 1 ? null : getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2Assets = node2Dir == null ? null : new File(node2Dir, "assets"); + final File node2ContextDir = node2Assets == null ? null : new File(node2Assets, paramContext.getId()); + if (node2ContextDir != null) { + assertTrue(node2ContextDir.exists()); + } + + // List assets and verify the expected asset is returned + final AssetsEntity assetListing = assertAssetListing(paramContext.getId(), 1); + assertAssetExists(asset, assetListing); + + // Attempt to delete the asset should be prevented since it is still referenced + assertThrows(NiFiClientException.class, () -> getNifiClient().getParamContextClient().deleteAsset(paramContext.getId(), asset.getAsset().getId())); + + // Change the parameter so that it no longer references the asset + final ParameterContextUpdateRequestEntity removeAssetUpdateRequest = getClientUtil().updateParameterContext(paramContext, Map.of("fileToIngest", "invalid")); + + // Wait for the update to complete + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), removeAssetUpdateRequest.getRequest().getRequestId()); + + // Attempt to delete the asset should now succeed since no longer referenced + final AssetEntity deletedAssetEntity = getNifiClient().getParamContextClient().deleteAsset(paramContext.getId(), asset.getAsset().getId()); + assertNotNull(deletedAssetEntity); + assertNotNull(deletedAssetEntity.getAsset()); + assertEquals(asset.getAsset().getId(), deletedAssetEntity.getAsset().getId()); + + // Ensure that the directories no longer exist + waitFor(() -> !node1ContextDir.exists()); + + if (node2ContextDir != null) { + waitFor(() -> !node2ContextDir.exists()); + } + + // Ensure that listing is now empty + assertAssetListing(paramContext.getId(), 0); + + getClientUtil().stopProcessor(ingest); + waitForStoppedProcessor(ingest.getId()); + } + + @Test + public void testAssetReferenceFromDifferentContext() throws NiFiClientException, IOException, InterruptedException { + // Create first context + final ParameterContextEntity paramContext1 = getClientUtil().createParameterContext("testAssetReferenceFirstContext", + Map.of("name", "foo", "fileToIngest", "")); + + // Create asset in first context + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext1.getId(), assetFile); + + // Update parameter in first context to reference asset in first context + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext1, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext1.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Create second context and try to update a parameter to reference the asset from first context + final ParameterContextEntity paramContext2 = getClientUtil().createParameterContext("testAssetReferenceSecondContext", Map.of("otherParam", "")); + assertThrows(NiFiClientException.class, () -> getClientUtil().updateParameterAssetReferences(paramContext2, Map.of("otherParam", List.of(asset.getAsset().getId())))); + } + + @Test + public void testAssetsRemovedWhenDeletingParameterContext() throws NiFiClientException, IOException, InterruptedException { + // Create context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testAssetsRemovedWhenDeletingParameterContext", + Map.of("name", "foo", "fileToIngest", "")); + + // Create asset in context + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetFile); + + // Update parameter to reference asset + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Check that the file exists in the assets directory. + final File node1Dir = getNumberOfNodes() == 1 ? getNiFiInstance().getInstanceDirectory() : getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1Assets = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1Assets, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNumberOfNodes() == 1 ? null : getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2Assets = node2Dir == null ? null : new File(node2Dir, "assets"); + final File node2ContextDir = node2Assets == null ? null : new File(node2Assets, paramContext.getId()); + if (node2ContextDir != null) { + assertTrue(node2ContextDir.exists()); + } + + // Delete context + final ParameterContextEntity latestParameterContext = getNifiClient().getParamContextClient().getParamContext(paramContext.getId(), false); + getNifiClient().getParamContextClient().deleteParamContext(paramContext.getId(), String.valueOf(latestParameterContext.getRevision().getVersion())); + + // Verify the directory for the context's assets was removed + assertFalse(node1ContextDir.exists()); + if (node2ContextDir != null) { + assertFalse(node2ContextDir.exists()); + } + } + + @Test + public void testAssetsRemainWhenRemovingReference() throws NiFiClientException, IOException, InterruptedException { + // Create context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testAssetsRemovedWhenDeletingParameterContext", + Map.of("name", "foo", "fileToIngest", "")); + + // Create asset + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetFile); + + // Update parameter to reference the asset + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Check that the file exists in the assets directory. + final File node1Dir = getNumberOfNodes() == 1 ? getNiFiInstance().getInstanceDirectory() : getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1Assets = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1Assets, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNumberOfNodes() == 1 ? null : getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2Assets = node2Dir == null ? null : new File(node2Dir, "assets"); + final File node2ContextDir = node2Assets == null ? null : new File(node2Assets, paramContext.getId()); + if (node2ContextDir != null) { + assertTrue(node2ContextDir.exists()); + } + + // Update the parameter to no longer reference the asset + final ParameterContextUpdateRequestEntity removeReferenceUpdateRequest = getClientUtil().updateParameterAssetReferences(paramContext, Map.of("fileToIngest", List.of())); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), removeReferenceUpdateRequest.getRequest().getRequestId()); + + // Verify the directory for the context's assets was not removed + assertTrue(node1ContextDir.exists()); + if (node2ContextDir != null) { + assertTrue(node2ContextDir.exists()); + } + } + + @Test + public void testUnreferencedAssetsRemovedWhenDeletingParameterContext() throws NiFiClientException, IOException, InterruptedException { + // Create context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testUnreferencedAssetsRemovedWhenDeletingParameterContext", + Map.of("name", "foo", "fileToIngest", "")); + + // Upload asset + final File assetFile = new File("src/test/resources/sample-assets/helloworld.txt"); + createAsset(paramContext.getId(), assetFile); + + // Check that the files exist in the assets directory + final File node1Dir = getNumberOfNodes() == 1 ? getNiFiInstance().getInstanceDirectory() : getNiFiInstance().getNodeInstance(1).getInstanceDirectory(); + final File node1Assets = new File(node1Dir, "assets"); + final File node1ContextDir = new File(node1Assets, paramContext.getId()); + assertTrue(node1ContextDir.exists()); + + final File node2Dir = getNumberOfNodes() == 1 ? null : getNiFiInstance().getNodeInstance(2).getInstanceDirectory(); + final File node2Assets = node2Dir == null ? null : new File(node2Dir, "assets"); + final File node2ContextDir = node2Assets == null ? null : new File(node2Assets, paramContext.getId()); + if (node2ContextDir != null) { + assertTrue(node2ContextDir.exists()); + } + + // Delete context + final ParameterContextEntity latestParameterContext = getNifiClient().getParamContextClient().getParamContext(paramContext.getId(), false); + getNifiClient().getParamContextClient().deleteParamContext(paramContext.getId(), String.valueOf(latestParameterContext.getRevision().getVersion())); + + // Verify the directory for the context's assets was removed + assertFalse(node1ContextDir.exists()); + if (node2ContextDir != null) { + assertFalse(node2ContextDir.exists()); + } + } + + @Test + public void testAssetReplacement() throws NiFiClientException, IOException, InterruptedException { + // Create context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testAssetReplacement", Map.of("name", "foo", "fileToIngest", "")); + + // Set the Parameter Context on the root Process Group + setParameterContext("root", paramContext); + + // Create a Processor and update it to reference a parameter + final ProcessorEntity ingest = getClientUtil().createProcessor("IngestFile"); + getClientUtil().updateProcessorProperties(ingest, Map.of("Filename", "#{fileToIngest}", "Delete File", "false")); + getClientUtil().updateProcessorSchedulingPeriod(ingest, "10 mins"); + + // Create an asset + final String assetName = "helloworld.txt"; + final File assetFile1 = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetName, assetFile1); + + // Update the parameter to reference the asset + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Connect the ingest processor to terminate processor and produce flow files + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(ingest, terminate, "success"); + waitForValidProcessor(ingest.getId()); + + // Run the flow and verify the flow files contain the contents of the asset + getClientUtil().startProcessor(ingest); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + final String contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertEquals("Hello, World!", contents); + + // Stop ingest processor and clear queue + getClientUtil().stopProcessor(ingest); + getClientUtil().waitForStoppedProcessor(ingest.getId()); + + getClientUtil().startProcessor(terminate); + waitForQueueCount(connection.getId(), 0); + + getClientUtil().stopProcessor(terminate); + getClientUtil().waitForStoppedProcessor(terminate.getId()); + + // Replace the asset by uploading a different file with the same name + final File assetFile2 = new File("src/test/resources/sample-assets/helloworld2.txt"); + final AssetEntity replacedAsset = createAsset(paramContext.getId(), assetName, assetFile2); + assertAsset(replacedAsset, assetName); + + // Run the flow again and verify the flow files contain the updated contents of the asset + getClientUtil().startProcessor(ingest); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + + final String contents2 = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertEquals("Hello, World! 2", contents2); + + getClientUtil().stopProcessor(ingest); + waitForStoppedProcessor(ingest.getId()); + } + + @Test + public void testAssetReferenceAfterRestart() throws NiFiClientException, IOException, InterruptedException { + // Create Parameter Context + final ParameterContextEntity paramContext = getClientUtil().createParameterContext("testAssetReferenceAfterRestart", + Map.of("name", "foo", "fileToIngest", "")); + + // Set the Parameter Context on the root Process Group + setParameterContext("root", paramContext); + + // Create a Processor and update it to reference a parameter + final ProcessorEntity ingest = getClientUtil().createProcessor("IngestFile"); + getClientUtil().updateProcessorProperties(ingest, Map.of("Filename", "#{fileToIngest}", "Delete File", "false")); + getClientUtil().updateProcessorSchedulingPeriod(ingest, "10 mins"); + + // Create an asset + final File assetFile1 = new File("src/test/resources/sample-assets/helloworld.txt"); + final AssetEntity asset = createAsset(paramContext.getId(), assetFile1); + + // Uupdate the parameter to reference the asset + final ParameterContextUpdateRequestEntity referenceAssetUpdateRequest = getClientUtil().updateParameterAssetReferences( + paramContext, Map.of("fileToIngest", List.of(asset.getAsset().getId()))); + getClientUtil().waitForParameterContextRequestToComplete(paramContext.getId(), referenceAssetUpdateRequest.getRequest().getRequestId()); + + // Ensure that Asset References are kept intact after restart + final boolean clustered = getNumberOfNodes() > 1; + if (clustered) { + disconnectNode(2); + } + final NiFiInstance restartNode = clustered ? getNiFiInstance().getNodeInstance(2) : getNiFiInstance(); + restartNode.stop(); + restartNode.start(true); + if (clustered) { + reconnectNode(2); + waitForAllNodesConnected(); + } + + final ProcessorEntity terminate = getClientUtil().createProcessor("TerminateFlowFile"); + final ConnectionEntity connection = getClientUtil().createConnection(ingest, terminate, "success"); + waitForValidProcessor(ingest.getId()); + + // Get the new Processor Entity because the revision will be reset on restart + final ProcessorEntity ingestAfterRestart = getNifiClient().getProcessorClient().getProcessor(ingest.getId()); + getClientUtil().startProcessor(ingestAfterRestart); + waitForQueueCount(connection.getId(), getNumberOfNodes()); + + final String contents = getClientUtil().getFlowFileContentAsUtf8(connection.getId(), 0); + assertEquals("Hello, World!", contents); + + getClientUtil().stopProcessor(ingest); + waitForStoppedProcessor(ingest.getId()); + } private Map waitForCounter(final String context, final String counterName, final long expectedValue) throws NiFiClientException, IOException, InterruptedException { return getClientUtil().waitForCounter(context, counterName, expectedValue); @@ -715,7 +1045,7 @@ public ParameterContextEntity createParameterContextEntity(final String name, fi return createParameterContextEntity(name, description, parameters, Collections.emptyList(), null, null); } - private ProcessGroupEntity setParameterContext(final String groupId, final ParameterContextEntity parameterContext) throws NiFiClientException, IOException { + protected ProcessGroupEntity setParameterContext(final String groupId, final ParameterContextEntity parameterContext) throws NiFiClientException, IOException { return getClientUtil().setParameterContext(groupId, parameterContext); } @@ -759,7 +1089,7 @@ public void fetchAndWaitForAppliedParameters(final ParameterProviderEntity entit waitForAppliedParameters(request); } - private void waitForValidProcessor(String id) throws InterruptedException, IOException, NiFiClientException { + void waitForValidProcessor(String id) throws InterruptedException, IOException, NiFiClientException { getClientUtil().waitForValidProcessor(id); } @@ -774,4 +1104,43 @@ private void waitForRunningProcessor(final String processorId) throws Interrupte private void waitForStoppedProcessor(final String processorId) throws InterruptedException, IOException, NiFiClientException { getClientUtil().waitForStoppedProcessor(processorId); } + + + protected AssetEntity createAsset(final String paramContextId, final File assetFile) throws NiFiClientException, IOException { + return createAsset(paramContextId, assetFile.getName(), assetFile); + } + + protected AssetEntity createAsset(final String paramContextId, final String assetName, final File assetFile) throws NiFiClientException, IOException { + final AssetEntity asset = getNifiClient().getParamContextClient().createAsset(paramContextId, assetName, assetFile); + logger.info("Created asset [{}] in parameter context [{}]", assetName, paramContextId); + assertAsset(asset, assetName); + return asset; + } + + protected AssetsEntity assertAssetListing(final String paramContextId, final int expectedCount) throws NiFiClientException, IOException { + final AssetsEntity assetListing = getNifiClient().getParamContextClient().getAssets(paramContextId); + assertNotNull(assetListing); + assertNotNull(assetListing.getAssets()); + assertEquals(expectedCount, assetListing.getAssets().size()); + return assetListing; + } + + protected void assertAssetExists(final AssetEntity asset, final AssetsEntity assets) { + final AssetEntity assetFromListing = assets.getAssets().stream() + .filter(a -> a.getAsset().getId().equals(asset.getAsset().getId())) + .findFirst() + .orElse(null); + assertNotNull(assetFromListing); + } + + protected void assertAsset(final AssetEntity asset, final String expectedName) { + assertNotNull(asset); + assertNotNull(asset.getAsset()); + assertNotNull(asset.getAsset().getId()); + assertNotNull(asset.getAsset().getDigest()); + assertNotNull(asset.getAsset().getMissingContent()); + assertFalse(asset.getAsset().getMissingContent()); + assertEquals(expectedName, asset.getAsset().getName()); + } + } diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties index 9433de39e05c..37002a6b1a14 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node1/nifi.properties @@ -128,6 +128,9 @@ nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties nifi.nar.persistence.provider.properties.directory=./nar_repository +# Asset Management +nifi.asset.manager.properties.directory=./assets + # Site to Site properties nifi.remote.input.host= nifi.remote.input.secure=false diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties index b5af42ff734b..3eeb24c7e4c9 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/clustered/node2/nifi.properties @@ -128,6 +128,9 @@ nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties nifi.nar.persistence.provider.properties.directory=./nar_repository +# Asset Management +nifi.asset.manager.properties.directory=./assets + # Site to Site properties nifi.remote.input.host= nifi.remote.input.secure=false diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties index c5a63d0e567c..1733299ead39 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/default/nifi.properties @@ -129,6 +129,9 @@ nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties nifi.nar.persistence.provider.properties.directory=./nar_repository +# Asset Management +nifi.asset.manager.properties.directory=./assets + # Site to Site properties nifi.remote.input.host= nifi.remote.input.secure=false diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties index 761b47cdcef7..93074ed666f3 100644 --- a/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/conf/pythonic/nifi.properties @@ -133,6 +133,9 @@ nifi.components.status.snapshot.frequency=1 min # NAR Persistence Provider Properties nifi.nar.persistence.provider.properties.directory=./nar_repository +# Asset Management +nifi.asset.manager.properties.directory=./assets + # Site to Site properties nifi.remote.input.host= nifi.remote.input.secure=false diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld.txt b/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld.txt new file mode 100644 index 000000000000..b45ef6fec895 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld.txt @@ -0,0 +1 @@ +Hello, World! \ No newline at end of file diff --git a/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld2.txt b/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld2.txt new file mode 100644 index 000000000000..3fd293c91a21 --- /dev/null +++ b/nifi-system-tests/nifi-system-test-suite/src/test/resources/sample-assets/helloworld2.txt @@ -0,0 +1 @@ +Hello, World! 2 \ No newline at end of file diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java index 607a264bca13..3d522dfc0bbe 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ParamContextClient.java @@ -16,11 +16,15 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextsEntity; +import java.io.File; import java.io.IOException; +import java.nio.file.Path; public interface ParamContextClient { @@ -40,4 +44,13 @@ public interface ParamContextClient { ParameterContextUpdateRequestEntity deleteParamContextUpdateRequest(String contextId, String updateRequestId) throws NiFiClientException, IOException; + AssetEntity createAsset(String contextId, String assetName, File file) throws NiFiClientException, IOException; + + AssetsEntity getAssets(String contextId) throws NiFiClientException, IOException; + + Path getAssetContent(String contextId, String assetId, File outputDirectory) throws NiFiClientException, IOException; + + AssetEntity deleteAsset(String contextId, String assetId) throws NiFiClientException, IOException; + + AssetEntity deleteAsset(String contextId, String assetId, boolean disconnectedNodeAcknowledged) throws NiFiClientException, IOException; } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java index 0a2aba22d104..456df8ed880c 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/ProcessGroupClient.java @@ -42,6 +42,8 @@ ProcessGroupEntity createProcessGroup(String parentGroupdId, ProcessGroupEntity ProcessGroupEntity updateProcessGroup(ProcessGroupEntity entity) throws NiFiClientException, IOException; + ProcessGroupEntity deleteProcessGroup(ProcessGroupEntity entity) throws NiFiClientException, IOException; + ControllerServiceEntity createControllerService(String processGroupId, ControllerServiceEntity controllerService) throws NiFiClientException, IOException; diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java index 0b25c6540796..8e401c524346 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyParamContextClient.java @@ -16,18 +16,28 @@ */ package org.apache.nifi.toolkit.cli.impl.client.nifi.impl; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import org.apache.commons.lang3.StringUtils; import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; import org.apache.nifi.toolkit.cli.impl.client.nifi.RequestConfig; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; import org.apache.nifi.web.api.entity.ParameterContextEntity; import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; import org.apache.nifi.web.api.entity.ParameterContextsEntity; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.WebTarget; -import jakarta.ws.rs.core.MediaType; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; public class JerseyParamContextClient extends AbstractJerseyClient implements ParamContextClient { @@ -164,4 +174,101 @@ public ParameterContextUpdateRequestEntity deleteParamContextUpdateRequest(final return getRequestBuilder(target).delete(ParameterContextUpdateRequestEntity.class); }); } + + @Override + public AssetEntity createAsset(final String contextId, final String assetName, final File file) throws NiFiClientException, IOException { + if (StringUtils.isBlank(contextId)) { + throw new IllegalArgumentException("Parameter context id cannot be null or blank"); + } + if (StringUtils.isBlank(assetName)) { + throw new IllegalArgumentException("Asset name cannot be null or blank"); + } + if (file == null) { + throw new IllegalArgumentException("File cannot be null"); + } + if (!file.exists()) { + throw new FileNotFoundException(file.getAbsolutePath()); + } + + try (final InputStream assetInputStream = new FileInputStream(file)) { + return executeAction("Error Creating Asset " + assetName + " for Parameter Context " + contextId, () -> { + final WebTarget target = paramContextTarget.path("{context-id}/assets") + .resolveTemplate("context-id", contextId); + + return getRequestBuilder(target) + .header("Filename", assetName) + .post( + Entity.entity(assetInputStream, MediaType.APPLICATION_OCTET_STREAM_TYPE), + AssetEntity.class + ); + }); + } + } + + @Override + public AssetsEntity getAssets(final String contextId) throws NiFiClientException, IOException { + if (StringUtils.isBlank(contextId)) { + throw new IllegalArgumentException("Parameter context id cannot be null or blank"); + } + return executeAction("Error retrieving parameter context assets", () -> { + final WebTarget target = paramContextTarget.path("{context-id}/assets") + .resolveTemplate("context-id", contextId); + return getRequestBuilder(target).get(AssetsEntity.class); + }); + } + + @Override + public Path getAssetContent(final String contextId, final String assetId, final File outputDirectory) + throws NiFiClientException, IOException { + if (StringUtils.isBlank(contextId)) { + throw new IllegalArgumentException("Parameter context id cannot be null or blank"); + } + if (StringUtils.isBlank(assetId)) { + throw new IllegalArgumentException("Asset id cannot be null or blank"); + } + return executeAction("Error getting asset content", () -> { + final WebTarget target = paramContextTarget.path("{context-id}/assets/{asset-id}") + .resolveTemplate("context-id", contextId) + .resolveTemplate("asset-id", assetId); + + final Response response = getRequestBuilder(target) + .accept(MediaType.APPLICATION_OCTET_STREAM_TYPE) + .get(); + + final String filename = getContentDispositionFilename(response); + final File assetFile = new File(outputDirectory, filename); + + try (final InputStream responseInputStream = response.readEntity(InputStream.class)) { + Files.copy(responseInputStream, assetFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + return assetFile.toPath(); + } + }); + } + + @Override + public AssetEntity deleteAsset(final String contextId, final String assetId) throws NiFiClientException, IOException { + return deleteAsset(contextId, assetId, false); + } + + @Override + public AssetEntity deleteAsset(final String contextId, final String assetId, final boolean disconnectedNodeAcknowledged) + throws NiFiClientException, IOException { + if (StringUtils.isBlank(contextId)) { + throw new IllegalArgumentException("Parameter context id cannot be null or blank"); + } + if (StringUtils.isBlank(assetId)) { + throw new IllegalArgumentException("Asset id cannot be null or blank"); + } + return executeAction("Error deleting asset", () -> { + WebTarget target = paramContextTarget.path("{context-id}/assets/{asset-id}") + .resolveTemplate("context-id", contextId) + .resolveTemplate("asset-id", assetId); + + if (disconnectedNodeAcknowledged) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + + return getRequestBuilder(target).delete(AssetEntity.class); + }); + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java index 53dc53b2db4f..87ce433fb433 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/client/nifi/impl/JerseyProcessGroupClient.java @@ -119,7 +119,30 @@ public ProcessGroupEntity updateProcessGroup(final ProcessGroupEntity entity) }); } - @Override + @Override + public ProcessGroupEntity deleteProcessGroup(final ProcessGroupEntity entity) throws NiFiClientException, IOException { + if (entity == null) { + throw new IllegalArgumentException("Process group entity cannot be null"); + } + if (entity.getRevision() == null || entity.getRevision().getVersion() == null) { + throw new IllegalArgumentException("Process group revision cannot be null"); + } + + return executeAction("Error deleting process group", () -> { + WebTarget target = processGroupsTarget + .path("{id}") + .resolveTemplate("id", entity.getId()) + .queryParam("version", entity.getRevision().getVersion()); + + if (entity.isDisconnectedNodeAcknowledged() == Boolean.TRUE) { + target = target.queryParam("disconnectedNodeAcknowledged", "true"); + } + + return getRequestBuilder(target).delete(ProcessGroupEntity.class); + }); + } + + @Override public ControllerServiceEntity createControllerService( final String processGroupId, final ControllerServiceEntity controllerService) throws NiFiClientException, IOException { if (StringUtils.isBlank(processGroupId)) { diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java index 497234c04120..d53bd6053aab 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/CommandOption.java @@ -154,6 +154,10 @@ public enum CommandOption { NAR_FILE("nar", "narFile", "A NAR file to upload, must contain full path and filename", true, true), NAR_UPLOAD_TIMEOUT("npt", "narProcessing", "Number of seconds after which a parameter context update will timeout (default: 60, maximum: 600)", true), + // NiFi - Assets + ASSET_FILE("af", "assetFile", "A file containing the asset content, must contain full path and filename", true, true), + ASSET_ID("aid", "assetId", "The id of an asset which can be referenced from a parameter", true, false), + // Security related KEYSTORE("ks", "keystore", "A keystore to use for TLS/SSL connections", true), KEYSTORE_TYPE("kst", "keystoreType", "The type of key store being used such as PKCS12", true), diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java index eb59030d2e94..cca3b4d8a239 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/NiFiCommandGroup.java @@ -56,8 +56,14 @@ import org.apache.nifi.toolkit.cli.impl.command.nifi.nodes.GetNode; import org.apache.nifi.toolkit.cli.impl.command.nifi.nodes.GetNodes; import org.apache.nifi.toolkit.cli.impl.command.nifi.nodes.OffloadNode; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.AddAssetReference; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.CreateAsset; import org.apache.nifi.toolkit.cli.impl.command.nifi.params.CreateParamContext; import org.apache.nifi.toolkit.cli.impl.command.nifi.params.CreateParamProvider; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.DeleteAsset; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.GetAsset; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.ListAssets; +import org.apache.nifi.toolkit.cli.impl.command.nifi.params.RemoveAssetReference; import org.apache.nifi.toolkit.cli.impl.command.nifi.params.DeleteParam; import org.apache.nifi.toolkit.cli.impl.command.nifi.params.DeleteParamContext; import org.apache.nifi.toolkit.cli.impl.command.nifi.params.DeleteParamProvider; @@ -212,6 +218,12 @@ protected List createCommands() { commands.add(new ListNars()); commands.add(new ListNarComponentTypes()); commands.add(new DeleteNar()); + commands.add(new CreateAsset()); + commands.add(new ListAssets()); + commands.add(new GetAsset()); + commands.add(new DeleteAsset()); + commands.add(new AddAssetReference()); + commands.add(new RemoveAssetReference()); return new ArrayList<>(commands); } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AbstractUpdateParamContextCommand.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AbstractUpdateParamContextCommand.java index 74ac6729b70c..cf8cb6b84524 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AbstractUpdateParamContextCommand.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AbstractUpdateParamContextCommand.java @@ -23,10 +23,17 @@ import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; import org.apache.nifi.toolkit.cli.impl.command.CommandOption; import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.web.api.dto.ParameterContextDTO; +import org.apache.nifi.web.api.dto.ParameterDTO; +import org.apache.nifi.web.api.dto.RevisionDTO; import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextReferenceEntity; import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ParameterEntity; import java.io.IOException; +import java.util.Collections; +import java.util.List; import java.util.Properties; import java.util.concurrent.atomic.AtomicBoolean; @@ -99,4 +106,23 @@ protected int getUpdateTimeout(final Properties properties) { throw new RuntimeException(e.getMessage(), e); } } + + protected ParameterContextEntity createContextEntityForUpdate(final String paramContextId, final ParameterDTO parameterDTO, + final List inheritedParameterContexts, + final RevisionDTO paramContextRevision) { + final ParameterEntity parameterEntity = new ParameterEntity(); + parameterEntity.setParameter(parameterDTO); + + final ParameterContextDTO parameterContextDTO = new ParameterContextDTO(); + parameterContextDTO.setId(paramContextId); + parameterContextDTO.setParameters(Collections.singleton(parameterEntity)); + parameterContextDTO.setInheritedParameterContexts(inheritedParameterContexts); + + final ParameterContextEntity updatedParameterContextEntity = new ParameterContextEntity(); + updatedParameterContextEntity.setId(paramContextId); + updatedParameterContextEntity.setComponent(parameterContextDTO); + updatedParameterContextEntity.setRevision(paramContextRevision); + return updatedParameterContextEntity; + + } } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AddAssetReference.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AddAssetReference.java new file mode 100644 index 000000000000..6109d0321f36 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/AddAssetReference.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; +import org.apache.nifi.web.api.dto.ParameterContextDTO; +import org.apache.nifi.web.api.dto.ParameterDTO; +import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ParameterEntity; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.Optional; +import java.util.Properties; +import java.util.Set; + +public class AddAssetReference extends AbstractUpdateParamContextCommand { + + public AddAssetReference() { + super("add-asset-reference", VoidResult.class); + } + + @Override + public String getDescription() { + return "Adds an asset reference to a parameter. The parameter will be created if it does not already exist."; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + addOption(CommandOption.PARAM_NAME.createOption()); + addOption(CommandOption.PARAM_DESC.createOption()); + addOption(CommandOption.ASSET_ID.createOption()); + addOption(CommandOption.UPDATE_TIMEOUT.createOption()); + } + + @Override + public VoidResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final String paramName = getRequiredArg(properties, CommandOption.PARAM_NAME); + final String paramDescription = getArg(properties, CommandOption.PARAM_DESC); + final String assetId = getRequiredArg(properties, CommandOption.ASSET_ID); + final int updateTimeout = getUpdateTimeout(properties); + + // Ensure the context exists... + final ParamContextClient paramContextClient = client.getParamContextClient(); + final ParameterContextEntity existingParameterContextEntity = paramContextClient.getParamContext(paramContextId, false); + final ParameterContextDTO existingParameterContextDTO = existingParameterContextEntity.getComponent(); + + // Determine if this is an existing param or a new one... + final Optional existingParam = existingParameterContextDTO.getParameters().stream() + .map(ParameterEntity::getParameter) + .filter(p -> p.getName().equals(paramName)) + .findFirst(); + + // Construct the DTOs and entities for submitting the update + final ParameterDTO parameterDTO = existingParam.orElseGet(ParameterDTO::new); + parameterDTO.setName(paramName); + parameterDTO.setSensitive(false); + parameterDTO.setProvided(false); + parameterDTO.setValue(null); + + if (paramDescription != null) { + parameterDTO.setDescription(paramDescription); + } + + if (parameterDTO.getReferencedAssets() == null) { + parameterDTO.setReferencedAssets(new ArrayList<>()); + } + + final Set assetReferences = new HashSet<>(parameterDTO.getReferencedAssets()); + assetReferences.add(new AssetReferenceDTO(assetId)); + parameterDTO.getReferencedAssets().clear(); + parameterDTO.getReferencedAssets().addAll(assetReferences); + + // Submit the update request... + final ParameterContextEntity updatedParameterContextEntity = createContextEntityForUpdate(paramContextId, parameterDTO, + existingParameterContextDTO.getInheritedParameterContexts(), existingParameterContextEntity.getRevision()); + + final ParameterContextUpdateRequestEntity updateRequestEntity = paramContextClient.updateParamContext(updatedParameterContextEntity); + performUpdate(paramContextClient, updatedParameterContextEntity, updateRequestEntity, updateTimeout); + + if (isInteractive()) { + println(); + } + + return VoidResult.getInstance(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/CreateAsset.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/CreateAsset.java new file mode 100644 index 000000000000..166d16e44fe9 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/CreateAsset.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.StringResult; +import org.apache.nifi.web.api.entity.AssetEntity; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; + +public class CreateAsset extends AbstractNiFiCommand { + + public CreateAsset() { + super("create-asset", StringResult.class); + } + + @Override + public String getDescription() { + return "Creates an asset in a given parameter context which can be referenced in a parameter."; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + addOption(CommandOption.ASSET_FILE.createOption()); + } + + @Override + public StringResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final File assetFile = new File(getRequiredArg(properties, CommandOption.ASSET_FILE)); + + final ParamContextClient paramContextClient = client.getParamContextClient(); + final AssetEntity assetEntity = paramContextClient.createAsset(paramContextId, assetFile.getName(), assetFile); + return new StringResult(assetEntity.getAsset().getId(), isInteractive()); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteAsset.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteAsset.java new file mode 100644 index 000000000000..b914c65ee48d --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteAsset.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; + +import java.io.IOException; +import java.util.Properties; + +public class DeleteAsset extends AbstractNiFiCommand { + + public DeleteAsset() { + super("delete-asset", VoidResult.class); + } + + @Override + public String getDescription() { + return "Deletes an asset from a given parameter context"; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + addOption(CommandOption.ASSET_ID.createOption()); + } + + @Override + public VoidResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final String assetId = getRequiredArg(properties, CommandOption.ASSET_ID); + client.getParamContextClient().deleteAsset(paramContextId, assetId); + return VoidResult.getInstance(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteParam.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteParam.java index 41476ffef338..d62f3ab61c80 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteParam.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/DeleteParam.java @@ -70,11 +70,11 @@ public VoidResult doExecute(final NiFiClient client, final Properties properties // Determine if this is an existing param or a new one... final Optional existingParam = existingEntity.getComponent().getParameters().stream() - .map(p -> p.getParameter()) + .map(ParameterEntity::getParameter) .filter(p -> p.getName().equals(paramName)) .findFirst(); - if (!existingParam.isPresent()) { + if (existingParam.isEmpty()) { throw new NiFiClientException("Unable to delete parameter, no parameter found with name '" + paramName + "'"); } diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/GetAsset.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/GetAsset.java new file mode 100644 index 000000000000..30d394ff4399 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/GetAsset.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Properties; + +public class GetAsset extends AbstractNiFiCommand { + + public GetAsset() { + super("get-asset", VoidResult.class); + } + + @Override + public String getDescription() { + return "Retrieves the content of the given asset."; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + addOption(CommandOption.ASSET_ID.createOption()); + addOption(CommandOption.OUTPUT_DIR.createOption()); + } + + @Override + public VoidResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final String assetId = getRequiredArg(properties, CommandOption.ASSET_ID); + final File outputDir = new File(getRequiredArg(properties, CommandOption.OUTPUT_DIR)); + final Path assetFile = client.getParamContextClient().getAssetContent(paramContextId, assetId, outputDir); + if (isInteractive()) { + println(assetFile.toFile().getAbsolutePath()); + } + return VoidResult.getInstance(); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/ListAssets.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/ListAssets.java new file mode 100644 index 000000000000..a0ea281ebd13 --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/ListAssets.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.command.nifi.AbstractNiFiCommand; +import org.apache.nifi.toolkit.cli.impl.result.nifi.AssetsResult; +import org.apache.nifi.web.api.entity.AssetsEntity; + +import java.io.IOException; +import java.util.Properties; + +public class ListAssets extends AbstractNiFiCommand { + + public ListAssets() { + super("list-assets", AssetsResult.class); + } + + @Override + public String getDescription() { + return "Lists the assets in the given parameter context."; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + } + + @Override + public AssetsResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final ParamContextClient paramContextClient = client.getParamContextClient(); + final AssetsEntity assetsEntity = paramContextClient.getAssets(paramContextId); + return new AssetsResult(getResultType(properties), assetsEntity); + } +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/RemoveAssetReference.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/RemoveAssetReference.java new file mode 100644 index 000000000000..aa7c1d9b1a2f --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/RemoveAssetReference.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.command.nifi.params; + +import org.apache.commons.cli.MissingOptionException; +import org.apache.nifi.toolkit.cli.api.CommandException; +import org.apache.nifi.toolkit.cli.api.Context; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClient; +import org.apache.nifi.toolkit.cli.impl.client.nifi.NiFiClientException; +import org.apache.nifi.toolkit.cli.impl.client.nifi.ParamContextClient; +import org.apache.nifi.toolkit.cli.impl.command.CommandOption; +import org.apache.nifi.toolkit.cli.impl.result.VoidResult; +import org.apache.nifi.web.api.dto.AssetReferenceDTO; +import org.apache.nifi.web.api.dto.ParameterContextDTO; +import org.apache.nifi.web.api.dto.ParameterDTO; +import org.apache.nifi.web.api.entity.ParameterContextEntity; +import org.apache.nifi.web.api.entity.ParameterContextUpdateRequestEntity; +import org.apache.nifi.web.api.entity.ParameterEntity; + +import java.io.IOException; +import java.util.List; +import java.util.Properties; + +public class RemoveAssetReference extends AbstractUpdateParamContextCommand { + + public RemoveAssetReference() { + super("remove-asset-reference", VoidResult.class); + } + + @Override + public String getDescription() { + return "Removes an asset reference from a given parameter."; + } + + @Override + protected void doInitialize(Context context) { + super.doInitialize(context); + addOption(CommandOption.PARAM_CONTEXT_ID.createOption()); + addOption(CommandOption.PARAM_NAME.createOption()); + addOption(CommandOption.ASSET_ID.createOption()); + addOption(CommandOption.UPDATE_TIMEOUT.createOption()); + } + + @Override + public VoidResult doExecute(final NiFiClient client, final Properties properties) + throws NiFiClientException, IOException, MissingOptionException, CommandException { + + final String paramContextId = getRequiredArg(properties, CommandOption.PARAM_CONTEXT_ID); + final String paramName = getRequiredArg(properties, CommandOption.PARAM_NAME); + final String assetId = getRequiredArg(properties, CommandOption.ASSET_ID); + final int updateTimeout = getUpdateTimeout(properties); + + // Ensure the context exists... + final ParamContextClient paramContextClient = client.getParamContextClient(); + final ParameterContextEntity existingParameterContextEntity = paramContextClient.getParamContext(paramContextId, false); + final ParameterContextDTO existingParameterContextDTO = existingParameterContextEntity.getComponent(); + + // Find the existing parameter by name or throw an exception + final ParameterDTO existingParam = existingParameterContextDTO.getParameters().stream() + .map(ParameterEntity::getParameter) + .filter(p -> p.getName().equals(paramName)) + .findFirst() + .orElseThrow(() -> new NiFiClientException("Parameter does not exist with the given name")); + + // Remove the given assetId from the referenced assets, or throw an exception if not referenced + final List assetReferences = existingParam.getReferencedAssets(); + if (assetReferences == null) { + throw new NiFiClientException("Parameter does not reference any assets"); + } + + final AssetReferenceDTO assetReferenceDTO = new AssetReferenceDTO(assetId); + if (!assetReferences.contains(assetReferenceDTO)) { + throw new NiFiClientException("Parameter does not reference the given asset"); + } + assetReferences.remove(assetReferenceDTO); + existingParam.setValue(null); + + // Submit the update request... + final ParameterContextEntity updatedParameterContextEntity = createContextEntityForUpdate(paramContextId, existingParam, + existingParameterContextDTO.getInheritedParameterContexts(), existingParameterContextEntity.getRevision()); + + final ParameterContextUpdateRequestEntity updateRequestEntity = paramContextClient.updateParamContext(updatedParameterContextEntity); + performUpdate(paramContextClient, updatedParameterContextEntity, updateRequestEntity, updateTimeout); + + if (isInteractive()) { + println(); + } + + return VoidResult.getInstance(); + } + +} diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/SetParam.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/SetParam.java index 37f346c455a4..0cf197d22a7c 100644 --- a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/SetParam.java +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/command/nifi/params/SetParam.java @@ -32,7 +32,6 @@ import org.apache.nifi.web.api.entity.ParameterEntity; import java.io.IOException; -import java.util.Collections; import java.util.Objects; import java.util.Optional; import java.util.Properties; @@ -83,11 +82,11 @@ public VoidResult doExecute(final NiFiClient client, final Properties properties // Determine if this is an existing param or a new one... final Optional existingParam = existingParameterContextDTO.getParameters().stream() - .map(p -> p.getParameter()) + .map(ParameterEntity::getParameter) .filter(p -> p.getName().equals(paramName)) .findFirst(); - if (!existingParam.isPresent() && paramValue == null) { + if (existingParam.isEmpty() && paramValue == null) { throw new IllegalArgumentException("A parameter value is required when creating a new parameter"); } @@ -95,8 +94,8 @@ public VoidResult doExecute(final NiFiClient client, final Properties properties throw new IllegalArgumentException(String.format("Parameter value supplied for parameter [%s] is the same as its current value", paramName)); } - // Construct the objects for the update... - final ParameterDTO parameterDTO = existingParam.isPresent() ? existingParam.get() : new ParameterDTO(); + // Create/update the parameter object + final ParameterDTO parameterDTO = existingParam.orElseGet(ParameterDTO::new); parameterDTO.setName(paramName); if (paramValue != null) { @@ -112,21 +111,10 @@ public VoidResult doExecute(final NiFiClient client, final Properties properties } parameterDTO.setProvided(false); - final ParameterEntity parameterEntity = new ParameterEntity(); - parameterEntity.setParameter(parameterDTO); - - final ParameterContextDTO parameterContextDTO = new ParameterContextDTO(); - parameterContextDTO.setId(existingParameterContextEntity.getId()); - parameterContextDTO.setParameters(Collections.singleton(parameterEntity)); - - parameterContextDTO.setInheritedParameterContexts(existingParameterContextDTO.getInheritedParameterContexts()); - - final ParameterContextEntity updatedParameterContextEntity = new ParameterContextEntity(); - updatedParameterContextEntity.setId(paramContextId); - updatedParameterContextEntity.setComponent(parameterContextDTO); - updatedParameterContextEntity.setRevision(existingParameterContextEntity.getRevision()); - // Submit the update request... + final ParameterContextEntity updatedParameterContextEntity = createContextEntityForUpdate(paramContextId, parameterDTO, + existingParameterContextDTO.getInheritedParameterContexts(), existingParameterContextEntity.getRevision()); + final ParameterContextUpdateRequestEntity updateRequestEntity = paramContextClient.updateParamContext(updatedParameterContextEntity); performUpdate(paramContextClient, updatedParameterContextEntity, updateRequestEntity, updateTimeout); diff --git a/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/nifi/AssetsResult.java b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/nifi/AssetsResult.java new file mode 100644 index 000000000000..9e37719dbb3a --- /dev/null +++ b/nifi-toolkit/nifi-toolkit-cli/src/main/java/org/apache/nifi/toolkit/cli/impl/result/nifi/AssetsResult.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.toolkit.cli.impl.result.nifi; + +import org.apache.nifi.toolkit.cli.api.ResultType; +import org.apache.nifi.toolkit.cli.impl.result.AbstractWritableResult; +import org.apache.nifi.toolkit.cli.impl.result.writer.DynamicTableWriter; +import org.apache.nifi.toolkit.cli.impl.result.writer.Table; +import org.apache.nifi.toolkit.cli.impl.result.writer.TableWriter; +import org.apache.nifi.web.api.dto.AssetDTO; +import org.apache.nifi.web.api.entity.AssetEntity; +import org.apache.nifi.web.api.entity.AssetsEntity; + +import java.io.IOException; +import java.io.PrintStream; +import java.util.Collection; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; + +public class AssetsResult extends AbstractWritableResult { + + private final AssetsEntity assetsEntity; + + public AssetsResult(final ResultType resultType, final AssetsEntity assetsEntity) { + super(resultType); + this.assetsEntity = Objects.requireNonNull(assetsEntity); + } + + @Override + public AssetsEntity getResult() { + return assetsEntity; + } + + @Override + protected void writeSimpleResult(final PrintStream output) throws IOException { + final Collection assetEntities = assetsEntity.getAssets(); + if (assetEntities == null) { + return; + } + + final List assetDTOS = assetEntities.stream() + .map(AssetEntity::getAsset) + .sorted(Comparator.comparing(AssetDTO::getName)) + .toList(); + + final Table table = new Table.Builder() + .column("#", 3, 3, false) + .column("ID", 36, 36, false) + .column("Name", 5, 40, false) + .build(); + + for (int i = 0; i < assetDTOS.size(); i++) { + final AssetDTO assetDTO = assetDTOS.get(i); + table.addRow(String.valueOf(i + 1), assetDTO.getId(), assetDTO.getName()); + } + + final TableWriter tableWriter = new DynamicTableWriter(); + tableWriter.write(table, output); + } +}