From 45e22d6dc3f93e577f1c2458c8d65ea1c98fa756 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 28 Jun 2017 11:02:58 +1000 Subject: [PATCH 1/2] ElasticsearchNode should use DesiredPort from NodeConfiguration Share ElasticsearchVersion instances --- .../ConnectionPooling/Sniffing/RoleDetection.doc.cs | 2 +- .../Framework/Configuration/EnvironmentConfiguration.cs | 4 ++-- .../Configuration/Versions/ElasticsearchVersion.cs | 5 +++++ src/Tests/Framework/Configuration/YamlConfiguration.cs | 2 +- .../ManagedElasticsearch/Nodes/ElasticsearchNode.cs | 9 +++++---- .../ManagedElasticsearch/Nodes/NodeConfiguration.cs | 8 ++++---- .../Plugins/ElasticsearchPluginCollection.cs | 6 +++--- .../Tasks/ValidationTasks/ValidateRunningVersion.cs | 4 ++-- src/Tests/Framework/TestClient.cs | 2 +- 9 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/Tests/ClientConcepts/ConnectionPooling/Sniffing/RoleDetection.doc.cs b/src/Tests/ClientConcepts/ConnectionPooling/Sniffing/RoleDetection.doc.cs index ee7ebdc100e..0fd94ecd17d 100644 --- a/src/Tests/ClientConcepts/ConnectionPooling/Sniffing/RoleDetection.doc.cs +++ b/src/Tests/ClientConcepts/ConnectionPooling/Sniffing/RoleDetection.doc.cs @@ -355,7 +355,7 @@ protected override string[] AdditionalServerSettings { get { - var es = this.Node.Version > new ElasticsearchVersion("5.0.0-alpha2") ? "" : "es."; + var es = this.Node.Version > ElasticsearchVersion.GetOrAdd("5.0.0-alpha2") ? "" : "es."; return new[] { diff --git a/src/Tests/Framework/Configuration/EnvironmentConfiguration.cs b/src/Tests/Framework/Configuration/EnvironmentConfiguration.cs index 4259624437e..e61691a0081 100644 --- a/src/Tests/Framework/Configuration/EnvironmentConfiguration.cs +++ b/src/Tests/Framework/Configuration/EnvironmentConfiguration.cs @@ -11,7 +11,7 @@ public class EnvironmentConfiguration : TestConfigurationBase { public override bool TestAgainstAlreadyRunningElasticsearch { get; protected set; } = false; public override bool ForceReseed { get; protected set; } = true; - public override ElasticsearchVersion ElasticsearchVersion { get; protected set; } = new ElasticsearchVersion("5.0.0"); + public override ElasticsearchVersion ElasticsearchVersion { get; protected set; } = ElasticsearchVersion.GetOrAdd("5.0.0"); public override TestMode Mode { get; protected set; } = TestMode.Unit; public override string ClusterFilter { get; protected set; } public override string TestFilter { get; protected set; } @@ -23,7 +23,7 @@ public EnvironmentConfiguration() var version = Environment.GetEnvironmentVariable("NEST_INTEGRATION_VERSION"); if (!string.IsNullOrEmpty(version)) Mode = TestMode.Integration; - this.ElasticsearchVersion = new ElasticsearchVersion(string.IsNullOrWhiteSpace(version) ? "5.0.0" : version); + this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(string.IsNullOrWhiteSpace(version) ? "5.0.0" : version); this.ClusterFilter = Environment.GetEnvironmentVariable("NEST_INTEGRATION_CLUSTER"); this.TestFilter = Environment.GetEnvironmentVariable("NEST_TEST_FILTER"); } diff --git a/src/Tests/Framework/Configuration/Versions/ElasticsearchVersion.cs b/src/Tests/Framework/Configuration/Versions/ElasticsearchVersion.cs index 8c68938bbf6..358e5768d9d 100644 --- a/src/Tests/Framework/Configuration/Versions/ElasticsearchVersion.cs +++ b/src/Tests/Framework/Configuration/Versions/ElasticsearchVersion.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Linq; +using System.Net; using System.Threading; using System.Xml.Linq; using Nest; @@ -15,6 +16,7 @@ public class ElasticsearchVersion : Version private static readonly object _lock = new { }; private static readonly ConcurrentDictionary SnapshotVersions = new ConcurrentDictionary(); private static readonly string SonaTypeUrl = "https://oss.sonatype.org/content/repositories/snapshots/org/elasticsearch/distribution/zip/elasticsearch"; + private static readonly ConcurrentDictionary Versions = new ConcurrentDictionary(); private string RootUrl => this.IsSnapshot ? SonaTypeUrl @@ -127,5 +129,8 @@ public string DownloadUrl public bool IsSnapshot => this.Version?.ToLower().Contains("snapshot") ?? false; public override string ToString() => this.Version; + + public static ElasticsearchVersion GetOrAdd(string version) => + Versions.GetOrAdd(version, v => new ElasticsearchVersion(v)); } } diff --git a/src/Tests/Framework/Configuration/YamlConfiguration.cs b/src/Tests/Framework/Configuration/YamlConfiguration.cs index 05b3af5775c..48d8f83bfef 100644 --- a/src/Tests/Framework/Configuration/YamlConfiguration.cs +++ b/src/Tests/Framework/Configuration/YamlConfiguration.cs @@ -23,7 +23,7 @@ public YamlConfiguration(string configurationFile) .ToDictionary(ConfigName, ConfigValue); this.Mode = GetTestMode(config["mode"]); - this.ElasticsearchVersion = new ElasticsearchVersion(config["elasticsearch_version"]); + this.ElasticsearchVersion = ElasticsearchVersion.GetOrAdd(config["elasticsearch_version"]); this.ForceReseed = bool.Parse(config["force_reseed"]); this.TestAgainstAlreadyRunningElasticsearch = bool.Parse(config["test_against_already_running_elasticsearch"]); this.ClusterFilter = config.ContainsKey("cluster_filter") ? config["cluster_filter"] : null; diff --git a/src/Tests/Framework/ManagedElasticsearch/Nodes/ElasticsearchNode.cs b/src/Tests/Framework/ManagedElasticsearch/Nodes/ElasticsearchNode.cs index f8fff77539a..6f8df6f58cc 100644 --- a/src/Tests/Framework/ManagedElasticsearch/Nodes/ElasticsearchNode.cs +++ b/src/Tests/Framework/ManagedElasticsearch/Nodes/ElasticsearchNode.cs @@ -26,11 +26,12 @@ public class ElasticsearchNode : IDisposable private readonly NodeConfiguration _config; public ElasticsearchVersion Version => _config.ElasticsearchVersion; + public NodeFileSystem FileSystem { get; } public bool Started { get; private set; } - public int Port { get; private set; } = 9200; + public int Port { get; private set; } private bool RunningOnCI { get; } @@ -39,7 +40,7 @@ public ElasticsearchNode(NodeConfiguration config) this._config = config; this.FileSystem = config.FileSystem; this.RunningOnCI = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEAMCITY_VERSION")); - + this.Port = config.DesiredPort; if (this._config.RunIntegrationTests && !this._config.TestAgainstAlreadyRunningElasticsearch) return; } @@ -91,7 +92,7 @@ public void Start(string[] settings) { var subscription = Observable.Using(() => process, p => p.Start()) .Select(c => new ElasticsearchConsoleOut(this._config.ElasticsearchVersion, c.Error, c.Data)) - .Subscribe(s => this.HandleConsoleMessage(s, handle), e => { throw e; }, () => handle.Set()); + .Subscribe(s => this.HandleConsoleMessage(s, handle), e => throw e, () => handle.Set()); this._composite.Add(subscription); if (!handle.WaitOne(timeout, true)) @@ -133,7 +134,7 @@ private void HandleConsoleMessage(ElasticsearchConsoleOut consoleOut, XplatManua if (this.ProcessId == null && consoleOut.TryParseNodeInfo(out version, out pid)) { - var startedVersion = new ElasticsearchVersion(version); + var startedVersion = ElasticsearchVersion.GetOrAdd(version); this.ProcessId = pid; if (this.Version != startedVersion) throw new Exception($"Booted elasticsearch is version {startedVersion} but the test config dictates {this.Version}"); diff --git a/src/Tests/Framework/ManagedElasticsearch/Nodes/NodeConfiguration.cs b/src/Tests/Framework/ManagedElasticsearch/Nodes/NodeConfiguration.cs index 7e227ce934d..bee45977a32 100644 --- a/src/Tests/Framework/ManagedElasticsearch/Nodes/NodeConfiguration.cs +++ b/src/Tests/Framework/ManagedElasticsearch/Nodes/NodeConfiguration.cs @@ -56,9 +56,9 @@ public NodeConfiguration(ITestConfiguration configuration, ClusterBase cluster) this.DesiredPort = cluster.DesiredPort; var attr = v.Major >= 5 ? "attr." : ""; - var indexedOrStored = v > new ElasticsearchVersion("5.0.0-alpha1") ? "stored" : "indexed"; - var shieldOrSecurity = v > new ElasticsearchVersion("5.0.0-alpha1") ? "xpack.security" : "shield"; - var es = v > new ElasticsearchVersion("5.0.0-alpha2") ? "" : "es."; + var indexedOrStored = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha1") ? "stored" : "indexed"; + var shieldOrSecurity = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha1") ? "xpack.security" : "shield"; + var es = v > ElasticsearchVersion.GetOrAdd("5.0.0-alpha2") ? "" : "es."; var b = this.XPackEnabled.ToString().ToLowerInvariant(); var sslEnabled = this.EnableSsl.ToString().ToLowerInvariant(); this.DefaultNodeSettings = new List @@ -78,7 +78,7 @@ public NodeConfiguration(ITestConfiguration configuration, ClusterBase cluster) $"{es}{shieldOrSecurity}.authc.realms.pki1.enabled={sslEnabled}", }; - if (v >= new ElasticsearchVersion("5.4.0")) + if (v >= ElasticsearchVersion.GetOrAdd("5.4.0")) this.DefaultNodeSettings.Add($"{es}search.remote.connect=true"); } diff --git a/src/Tests/Framework/ManagedElasticsearch/Plugins/ElasticsearchPluginCollection.cs b/src/Tests/Framework/ManagedElasticsearch/Plugins/ElasticsearchPluginCollection.cs index 9733d588eaf..24b52fb3351 100644 --- a/src/Tests/Framework/ManagedElasticsearch/Plugins/ElasticsearchPluginCollection.cs +++ b/src/Tests/Framework/ManagedElasticsearch/Plugins/ElasticsearchPluginCollection.cs @@ -8,12 +8,12 @@ public class ElasticsearchPluginCollection : KeyedCollection version < new ElasticsearchVersion("5.0.0-alpha3")), + new ElasticsearchPluginConfiguration(ElasticsearchPlugin.DeleteByQuery, version => version < ElasticsearchVersion.GetOrAdd("5.0.0-alpha3")), new ElasticsearchPluginConfiguration(ElasticsearchPlugin.MapperAttachments), new ElasticsearchPluginConfiguration(ElasticsearchPlugin.MapperMurmer3), new ElasticsearchPluginConfiguration(ElasticsearchPlugin.XPack), - new ElasticsearchPluginConfiguration(ElasticsearchPlugin.IngestGeoIp, version => version >= new ElasticsearchVersion("5.0.0-alpha3")), - new ElasticsearchPluginConfiguration(ElasticsearchPlugin.IngestAttachment, version => version >= new ElasticsearchVersion("5.0.0-alpha3")), + new ElasticsearchPluginConfiguration(ElasticsearchPlugin.IngestGeoIp, version => version >= ElasticsearchVersion.GetOrAdd("5.0.0-alpha3")), + new ElasticsearchPluginConfiguration(ElasticsearchPlugin.IngestAttachment, version => version >= ElasticsearchVersion.GetOrAdd("5.0.0-alpha3")), new ElasticsearchPluginConfiguration(ElasticsearchPlugin.AnalysisKuromoji), new ElasticsearchPluginConfiguration(ElasticsearchPlugin.AnalysisIcu) }; diff --git a/src/Tests/Framework/ManagedElasticsearch/Tasks/ValidationTasks/ValidateRunningVersion.cs b/src/Tests/Framework/ManagedElasticsearch/Tasks/ValidationTasks/ValidateRunningVersion.cs index dea19e38ba7..047ed20d8f4 100644 --- a/src/Tests/Framework/ManagedElasticsearch/Tasks/ValidationTasks/ValidateRunningVersion.cs +++ b/src/Tests/Framework/ManagedElasticsearch/Tasks/ValidationTasks/ValidateRunningVersion.cs @@ -17,8 +17,8 @@ public override void Validate(IElasticClient client, NodeConfiguration configura if (!alreadyUp.IsValid) return; var v = configuration.ElasticsearchVersion; - var alreadyUpVersion = new ElasticsearchVersion(alreadyUp.Version.Number); - var alreadyUpSnapshotVersion = new ElasticsearchVersion(alreadyUp.Version.Number + "-SNAPSHOT"); + var alreadyUpVersion = ElasticsearchVersion.GetOrAdd(alreadyUp.Version.Number); + var alreadyUpSnapshotVersion = ElasticsearchVersion.GetOrAdd(alreadyUp.Version.Number + "-SNAPSHOT"); if (v != alreadyUpVersion && v != alreadyUpSnapshotVersion) throw new Exception($"running elasticsearch is version {alreadyUpVersion} but the test config dictates {v}"); } diff --git a/src/Tests/Framework/TestClient.cs b/src/Tests/Framework/TestClient.cs index 9093fa11c76..b9518e581ea 100644 --- a/src/Tests/Framework/TestClient.cs +++ b/src/Tests/Framework/TestClient.cs @@ -96,7 +96,7 @@ private static IClrTypeMapping ProjectMapping(ClrTypeMappingDescriptor< m.Ignore(p => p.Ranges); return m; } - public static string PercolatorType => Configuration.ElasticsearchVersion <= new ElasticsearchVersion("5.0.0-alpha1") + public static string PercolatorType => Configuration.ElasticsearchVersion <= ElasticsearchVersion.GetOrAdd("5.0.0-alpha1") ? ".percolator" : "query"; From 501da81405dc64d729d2dbd800a8f1f16d121a8a Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 28 Jun 2017 11:04:28 +1000 Subject: [PATCH 2/2] Allow more time for flaky integration tests Increase number of documents and wait time for BulkAll cancellation and dispose tests Increase request timeout for Watcher inline execution test --- src/Tests/Document/Multiple/BulkAll/BulkAllApiTests.cs | 8 ++++---- .../XPack/Watcher/ExecuteWatch/ExecuteWatchApiTests.cs | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/Tests/Document/Multiple/BulkAll/BulkAllApiTests.cs b/src/Tests/Document/Multiple/BulkAll/BulkAllApiTests.cs index 9e96985870c..c6076857f50 100644 --- a/src/Tests/Document/Multiple/BulkAll/BulkAllApiTests.cs +++ b/src/Tests/Document/Multiple/BulkAll/BulkAllApiTests.cs @@ -120,7 +120,7 @@ public void DisposingObservableCancelsBulkAll() var handle = new ManualResetEvent(false); var size = 1000; - var pages = 100; + var pages = 1000; var seenPages = 0; var numberOfDocuments = size * pages; var documents = this.CreateLazyStreamOfDocuments(numberOfDocuments); @@ -146,7 +146,7 @@ public void DisposingObservableCancelsBulkAll() observableBulk.Subscribe(bulkObserver); //we wait N seconds to see some bulks - handle.WaitOne(TimeSpan.FromSeconds(1)); + handle.WaitOne(TimeSpan.FromSeconds(3)); observableBulk.Dispose(); //we wait N seconds to give in flight request a chance to cancel handle.WaitOne(TimeSpan.FromSeconds(3)); @@ -165,7 +165,7 @@ public void CancelBulkAll() var handle = new ManualResetEvent(false); var size = 1000; - var pages = 100; + var pages = 1000; var seenPages = 0; var numberOfDocuments = size * pages; var documents = this.CreateLazyStreamOfDocuments(numberOfDocuments); @@ -191,7 +191,7 @@ public void CancelBulkAll() observableBulk.Subscribe(bulkObserver); //we wait Nseconds to see some bulks - handle.WaitOne(TimeSpan.FromSeconds(1)); + handle.WaitOne(TimeSpan.FromSeconds(3)); tokenSource.Cancel(); //we wait Nseconds to give in flight request a chance to cancel handle.WaitOne(TimeSpan.FromSeconds(3)); diff --git a/src/Tests/XPack/Watcher/ExecuteWatch/ExecuteWatchApiTests.cs b/src/Tests/XPack/Watcher/ExecuteWatch/ExecuteWatchApiTests.cs index 089ba663280..c1fc635822d 100644 --- a/src/Tests/XPack/Watcher/ExecuteWatch/ExecuteWatchApiTests.cs +++ b/src/Tests/XPack/Watcher/ExecuteWatch/ExecuteWatchApiTests.cs @@ -548,6 +548,9 @@ protected override LazyResponses ClientUsage() => Calls( .Body("{}") ) ) + ) + .RequestConfiguration(r => r + .RequestTimeout(TimeSpan.FromMinutes(2)) ); protected override ExecuteWatchRequest Initializer => @@ -615,6 +618,10 @@ protected override LazyResponses ClientUsage() => Calls( Method = HttpInputMethod.Post, Body = "{}" } + }, + RequestConfiguration = new RequestConfiguration + { + RequestTimeout = TimeSpan.FromMinutes(2) } };