Skip to content

Commit

Permalink
[Transform] Cleanup and simplify reading transform settings
Browse files Browse the repository at this point in the history
  • Loading branch information
przemekwitek committed Feb 15, 2024
1 parent fd17e0c commit cf2fb5b
Show file tree
Hide file tree
Showing 19 changed files with 305 additions and 107 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@
import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;

public class SettingsConfig implements Writeable, ToXContentObject {

public static final SettingsConfig EMPTY = new SettingsConfig(null, null, null, null, null, null, null, (Integer) null);

public static final ConstructingObjectParser<SettingsConfig, Void> STRICT_PARSER = createParser(false);
public static final ConstructingObjectParser<SettingsConfig, Void> LENIENT_PARSER = createParser(true);

Expand Down Expand Up @@ -110,10 +113,6 @@ private static ConstructingObjectParser<SettingsConfig, Void> createParser(boole
private final Integer numFailureRetries;
private final Integer unattended;

public SettingsConfig() {
this(null, null, (Integer) null, (Integer) null, (Integer) null, (Integer) null, (Integer) null, (Integer) null);
}

public SettingsConfig(
Integer maxPageSearchSize,
Float docsPerSecond,
Expand All @@ -136,7 +135,7 @@ public SettingsConfig(
);
}

SettingsConfig(
private SettingsConfig(
Integer maxPageSearchSize,
Float docsPerSecond,
Integer datesAsEpochMillis,
Expand Down Expand Up @@ -188,51 +187,51 @@ public Float getDocsPerSecond() {
return docsPerSecond;
}

public Boolean getDatesAsEpochMillis() {
Boolean getDatesAsEpochMillis() {
return datesAsEpochMillis != null ? datesAsEpochMillis > 0 : null;
}

public Integer getDatesAsEpochMillisForUpdate() {
Integer getDatesAsEpochMillisForUpdate() {
return datesAsEpochMillis;
}

public Boolean getAlignCheckpoints() {
Boolean getAlignCheckpoints() {
return alignCheckpoints != null ? (alignCheckpoints > 0) || (alignCheckpoints == DEFAULT_ALIGN_CHECKPOINTS) : null;
}

public Integer getAlignCheckpointsForUpdate() {
Integer getAlignCheckpointsForUpdate() {
return alignCheckpoints;
}

public Boolean getUsePit() {
Boolean getUsePit() {
return usePit != null ? (usePit > 0) || (usePit == DEFAULT_USE_PIT) : null;
}

public Integer getUsePitForUpdate() {
Integer getUsePitForUpdate() {
return usePit;
}

public Boolean getDeduceMappings() {
Boolean getDeduceMappings() {
return deduceMappings != null ? (deduceMappings > 0) || (deduceMappings == DEFAULT_DEDUCE_MAPPINGS) : null;
}

public Integer getDeduceMappingsForUpdate() {
Integer getDeduceMappingsForUpdate() {
return deduceMappings;
}

public Integer getNumFailureRetries() {
Integer getNumFailureRetries() {
return numFailureRetries != null ? (numFailureRetries == DEFAULT_NUM_FAILURE_RETRIES ? null : numFailureRetries) : null;
}

public Integer getNumFailureRetriesForUpdate() {
Integer getNumFailureRetriesForUpdate() {
return numFailureRetries;
}

public Boolean getUnattended() {
Boolean getUnattended() {
return unattended != null ? (unattended == DEFAULT_UNATTENDED) ? null : (unattended > 0) : null;
}

public Integer getUnattendedForUpdate() {
Integer getUnattendedForUpdate() {
return unattended;
}

Expand Down Expand Up @@ -495,7 +494,7 @@ public Builder setNumFailureRetries(Integer numFailureRetries) {
* An explicit `null` resets to default.
*
* @param unattended true if this is a unattended transform.
* @return the {@link Builder} with usePit set.
* @return the {@link Builder} with unattended set.
*/
public Builder setUnattended(Boolean unattended) {
this.unattended = unattended == null ? DEFAULT_UNATTENDED : unattended ? 1 : 0;
Expand Down Expand Up @@ -545,7 +544,6 @@ public Builder update(SettingsConfig update) {
if (update.getUnattendedForUpdate() != null) {
this.unattended = update.getUnattendedForUpdate().equals(DEFAULT_UNATTENDED) ? null : update.getUnattendedForUpdate();
}

return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ public TransformConfig(
this.pivotConfig = pivotConfig;
this.latestConfig = latestConfig;
this.description = description;
this.settings = settings == null ? new SettingsConfig() : settings;
this.settings = settings == null ? SettingsConfig.EMPTY : settings;
this.metadata = metadata;
this.retentionPolicyConfig = retentionPolicyConfig;
if (this.description != null && this.description.length() > MAX_DESCRIPTION_LENGTH) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.core.transform.transforms;

import org.elasticsearch.xpack.core.transform.TransformConfigVersion;

public final class TransformEffectiveSettings {

private TransformEffectiveSettings() {}

/**
* Determines if the transform should write dates as epoch millis based on settings and version.
*
* @param settings transform's settings
* @return whether or not the transform is unattended
*/
public static boolean writeDatesAsEpochMillis(SettingsConfig settings, TransformConfigVersion version) {
// defines how dates are written, if not specified in settings
// < 7.11 as epoch millis
// >= 7.11 as string
// note: it depends on the version when the transform has been created, not the version of the code
return settings.getDatesAsEpochMillis() != null
? settings.getDatesAsEpochMillis()
: version.before(TransformConfigVersion.V_7_11_0);
}

/**
* Determines if aligning checkpoints is disabled for this transform based on settings.
*
* @param settings transform's settings
* @return whether or not aligning checkpoints is disabled for this transform
*/
public static boolean isAlignCheckpointsDisabled(SettingsConfig settings) {
return Boolean.FALSE.equals(settings.getAlignCheckpoints());
}

/**
* Determines if pit is disabled for this transform based on settings.
*
* @param settings transform's settings
* @return whether or not pit is disabled for this transform
*/
public static boolean isPitDisabled(SettingsConfig settings) {
return Boolean.FALSE.equals(settings.getUsePit());
}

/**
* Determines if mappings deduction is disabled for this transform based on settings.
*
* @param settings transform's settings
* @return whether or not mappings deduction is disabled for this transform
*/
public static boolean isDeduceMappingsDisabled(SettingsConfig settings) {
return Boolean.FALSE.equals(settings.getDeduceMappings());
}

/**
* Determines the appropriate number of retries.
* <p>
* The number of retries are read from the config or if not read from the context which is based on a cluster wide default.
* If the transform runs in unattended mode, the number of retries is always indefinite.
*
* @param settings transform's settings
* @return the number of retries or -1 if retries are indefinite
*/
public static int getNumFailureRetries(SettingsConfig settings, int defaultNumFailureRetries) {
return isUnattended(settings) ? -1
: settings.getNumFailureRetries() != null ? settings.getNumFailureRetries()
: defaultNumFailureRetries;
}

/**
* Determines if the transform is unattended based on settings.
*
* @param settings transform's settings
* @return whether or not the transform is unattended
*/
public static boolean isUnattended(SettingsConfig settings) {
return Boolean.TRUE.equals(settings.getUnattended());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -33,32 +33,30 @@ public class SettingsConfigTests extends AbstractSerializingTransformTestCase<Se
private boolean lenient;

public static SettingsConfig randomSettingsConfig() {
Integer unattended = randomBoolean() ? null : randomIntBetween(0, 1);

Boolean unattended = randomBoolean() ? null : randomBoolean();
return new SettingsConfig(
randomBoolean() ? null : randomIntBetween(10, 10_000),
randomBoolean() ? null : randomFloat(),
randomBoolean() ? null : randomIntBetween(0, 1),
randomBoolean() ? null : randomIntBetween(0, 1),
randomBoolean() ? null : randomIntBetween(0, 1),
randomBoolean() ? null : randomIntBetween(0, 1),
randomBoolean() ? null : randomBoolean(),
randomBoolean() ? null : randomBoolean(),
randomBoolean() ? null : randomBoolean(),
randomBoolean() ? null : randomBoolean(),
// don't set retries if unattended is set to true
randomBoolean() ? null : Integer.valueOf(1).equals(unattended) ? null : randomIntBetween(-1, 100),
randomBoolean() ? null : Boolean.TRUE.equals(unattended) ? null : randomIntBetween(-1, 100),
unattended
);
}

public static SettingsConfig randomNonEmptySettingsConfig() {
Integer unattended = randomIntBetween(0, 1);

Boolean unattended = randomBoolean() ? null : randomBoolean();
return new SettingsConfig(
randomIntBetween(10, 10_000),
randomFloat(),
randomIntBetween(0, 1),
randomIntBetween(0, 1),
randomIntBetween(0, 1),
randomIntBetween(0, 1),
Integer.valueOf(1).equals(unattended) ? -1 : randomIntBetween(-1, 100),
randomBoolean(),
randomBoolean(),
randomBoolean(),
randomBoolean(),
Boolean.TRUE.equals(unattended) ? -1 : randomIntBetween(-1, 100),
unattended
);
}
Expand Down Expand Up @@ -94,7 +92,6 @@ protected Reader<SettingsConfig> instanceReader() {
}

public void testExplicitNullParsing() throws IOException {

// explicit null
assertThat(fromString("{\"max_page_search_size\" : null}").getMaxPageSearchSize(), equalTo(-1));
// not set
Expand All @@ -119,6 +116,11 @@ public void testExplicitNullParsing() throws IOException {
assertThat(fromString("{\"num_failure_retries\" : null}").getNumFailureRetriesForUpdate(), equalTo(-2));
assertNull(fromString("{}").getNumFailureRetries());
assertNull(fromString("{}").getNumFailureRetriesForUpdate());

assertNull(fromString("{\"unattended\" : null}").getUnattended());
assertThat(fromString("{\"unattended\" : null}").getUnattendedForUpdate(), equalTo(-1));
assertNull(fromString("{}").getUnattended());
assertNull(fromString("{}").getUnattendedForUpdate());
}

public void testUpdateMaxPageSearchSizeUsingBuilder() throws IOException {
Expand Down
Loading

0 comments on commit cf2fb5b

Please sign in to comment.