[mobile]Configure fallback resolver for cares (envoyproxy#36078)

Commit Message: Allow user to specify fallback resolvers when c-ares
fails to find resolver hosts from system configs.
Risk Level: low
Testing: unit test
Docs Changes: n/a
Release Notes: n/a
Platform Specific Features: android only

---------

Signed-off-by: Renjie Tang <renjietang@google.com>

mobile/library/cc/engine_builder.cc

@@ -56,6 +56,11 @@ EngineBuilder& EngineBuilder::setUseCares(bool use_cares) {
   use_cares_ = use_cares;
   return *this;
 }
+
+EngineBuilder& EngineBuilder::addCaresFallbackResolver(std::string host, int port) {
+  cares_fallback_resolvers_.emplace_back(std::move(host), port);
+  return *this;
+}
 #endif
 EngineBuilder& EngineBuilder::setLogLevel(Logger::Logger::Levels log_level) {
   log_level_ = log_level;
@@ -491,6 +496,14 @@ std::unique_ptr<envoy::config::bootstrap::v3::Bootstrap> EngineBuilder::generate
 #else
   if (use_cares_) {
     envoy::extensions::network::dns_resolver::cares::v3::CaresDnsResolverConfig resolver_config;
+    if (!cares_fallback_resolvers_.empty()) {
+      for (const auto& [host, port] : cares_fallback_resolvers_) {
+        auto* address = resolver_config.add_resolvers();
+        address->mutable_socket_address()->set_address(host);
+        address->mutable_socket_address()->set_port_value(port);
+      }
+      resolver_config.set_use_resolvers_as_fallback(true);
+    }
     dns_cache_config->mutable_typed_dns_resolver_config()->set_name(
         "envoy.network.dns_resolver.cares");
     dns_cache_config->mutable_typed_dns_resolver_config()->mutable_typed_config()->PackFrom(

mobile/library/cc/engine_builder.h

@@ -114,6 +114,7 @@ class EngineBuilder {
 #else
   // Only android supports c_ares
   EngineBuilder& setUseCares(bool use_cares);
+  EngineBuilder& addCaresFallbackResolver(std::string host, int port);
 #endif
 
   // This is separated from build() for the sake of testability
@@ -169,6 +170,7 @@ class EngineBuilder {
   bool enable_http3_ = true;
 #if !defined(__APPLE__)
   bool use_cares_ = false;
+  std::vector<std::pair<std::string, int>> cares_fallback_resolvers_;
 #endif
   std::string http3_connection_options_ = "";
   std::string http3_client_connection_options_ = "";

mobile/library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java

@@ -1,5 +1,6 @@
 package io.envoyproxy.envoymobile.engine;
 
+import android.util.Pair;
 import java.util.Collections;
 import java.util.List;
 import java.util.ArrayList;
@@ -36,6 +37,7 @@ public enum TrustChainVerification {
   public final boolean enableDrainPostDnsRefresh;
   public final boolean enableHttp3;
   public final boolean useCares;
+  public final List<Pair<String, String>> caresFallbackResolvers;
   public final boolean forceV6;
   public final boolean useGro;
   public final String http3ConnectionOptions;
@@ -126,6 +128,8 @@ public enum TrustChainVerification {
    * @param keyValueStores                                platform key-value store implementations.
    * @param enablePlatformCertificatesValidation          whether to use the platform verifier.
    * @param upstreamTlsSni                                the upstream TLS socket SNI override.
+   * @param caresFallbackResolvers                        A list of host port pair that's used as
+   *     c-ares's fallback resolvers.
    */
   public EnvoyConfiguration(
       int connectTimeoutSeconds, int dnsRefreshSeconds, int dnsFailureRefreshSecondsBase,
@@ -144,7 +148,8 @@ public EnvoyConfiguration(
       List<EnvoyHTTPFilterFactory> httpPlatformFilterFactories,
       Map<String, EnvoyStringAccessor> stringAccessors,
       Map<String, EnvoyKeyValueStore> keyValueStores, Map<String, Boolean> runtimeGuards,
-      boolean enablePlatformCertificatesValidation, String upstreamTlsSni) {
+      boolean enablePlatformCertificatesValidation, String upstreamTlsSni,
+      List<Pair<String, Integer>> caresFallbackResolvers) {
     JniLibrary.load();
     this.connectTimeoutSeconds = connectTimeoutSeconds;
     this.dnsRefreshSeconds = dnsRefreshSeconds;
@@ -159,6 +164,11 @@ public EnvoyConfiguration(
     this.enableDrainPostDnsRefresh = enableDrainPostDnsRefresh;
     this.enableHttp3 = enableHttp3;
     this.useCares = useCares;
+    this.caresFallbackResolvers = new ArrayList<>();
+    for (Pair<String, Integer> hostAndPort : caresFallbackResolvers) {
+      this.caresFallbackResolvers.add(
+          new Pair<String, String>(hostAndPort.first, String.valueOf(hostAndPort.second)));
+    }
     this.forceV6 = forceV6;
     this.useGro = useGro;
     this.http3ConnectionOptions = http3ConnectionOptions;
@@ -215,6 +225,8 @@ public long createBootstrap() {
     byte[][] runtimeGuards = JniBridgeUtility.mapToJniBytes(this.runtimeGuards);
     byte[][] quicHints = JniBridgeUtility.mapToJniBytes(this.quicHints);
     byte[][] quicSuffixes = JniBridgeUtility.stringsToJniBytes(quicCanonicalSuffixes);
+    byte[][] caresFallbackResolvers =
+        JniBridgeUtility.listOfStringPairsToJniBytes(this.caresFallbackResolvers);
 
     return JniLibrary.createBootstrap(
         connectTimeoutSeconds, dnsRefreshSeconds, dnsFailureRefreshSecondsBase,
@@ -226,6 +238,6 @@ public long createBootstrap() {
         h2ConnectionKeepaliveIdleIntervalMilliseconds, h2ConnectionKeepaliveTimeoutSeconds,
         maxConnectionsPerHost, streamIdleTimeoutSeconds, perTryIdleTimeoutSeconds, appVersion,
         appId, enforceTrustChainVerification, filterChain, enablePlatformCertificatesValidation,
-        upstreamTlsSni, runtimeGuards);
+        upstreamTlsSni, runtimeGuards, caresFallbackResolvers);
   }
 }

mobile/library/java/io/envoyproxy/envoymobile/engine/JniBridgeUtility.java

@@ -1,5 +1,6 @@
 package io.envoyproxy.envoymobile.engine;
 
+import android.util.Pair;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
@@ -57,6 +58,15 @@ public static byte[][] mapToJniBytes(Map<String, String> stringMap) {
     return convertedBytes.toArray(new byte[0][0]);
   }
 
+  public static byte[][] listOfStringPairsToJniBytes(List<Pair<String, String>> stringList) {
+    final List<byte[]> convertedBytes = new ArrayList<byte[]>(stringList.size() * 2);
+    for (Pair<String, String> entry : stringList) {
+      convertedBytes.add(entry.first.getBytes(StandardCharsets.UTF_8));
+      convertedBytes.add(entry.second.getBytes(StandardCharsets.UTF_8));
+    }
+    return convertedBytes.toArray(new byte[0][0]);
+  }
+
   public static byte[][] toJniTags(Map<String, String> tags) {
     if (tags == null) {
       return null;

mobile/library/java/io/envoyproxy/envoymobile/engine/JniLibrary.java

@@ -311,7 +311,8 @@ public static native long createBootstrap(
       long h2ConnectionKeepaliveIdleIntervalMilliseconds, long h2ConnectionKeepaliveTimeoutSeconds,
       long maxConnectionsPerHost, long streamIdleTimeoutSeconds, long perTryIdleTimeoutSeconds,
       String appVersion, String appId, boolean trustChainVerification, byte[][] filterChain,
-      boolean enablePlatformCertificatesValidation, String upstreamTlsSni, byte[][] runtimeGuards);
+      boolean enablePlatformCertificatesValidation, String upstreamTlsSni, byte[][] runtimeGuards,
+      byte[][] cares_fallback_resolvers);
 
   /**
    * Initializes c-ares.

mobile/library/java/org/chromium/net/impl/NativeCronvoyEngineBuilderImpl.java

@@ -4,6 +4,7 @@
 
 import java.nio.charset.StandardCharsets;
 import android.content.Context;
+import android.util.Pair;
 import androidx.annotation.VisibleForTesting;
 import com.google.protobuf.Struct;
 import io.envoyproxy.envoymobile.engine.AndroidEngineImpl;
@@ -50,6 +51,7 @@ public class NativeCronvoyEngineBuilderImpl extends CronvoyEngineBuilderImpl {
   private final List<String> mDnsFallbackNameservers = Collections.emptyList();
   private final boolean mEnableDnsFilterUnroutableFamilies = true;
   private boolean mUseCares = false;
+  private final List<Pair<String, Integer>> mCaresFallbackResolvers = new ArrayList<>();
   private boolean mForceV6 = true;
   private boolean mUseGro = false;
   private boolean mEnableDrainPostDnsRefresh = false;
@@ -98,6 +100,17 @@ public NativeCronvoyEngineBuilderImpl setUseCares(boolean enable) {
     return this;
   }
 
+  /**
+   * Add a fallback resolver to c_cares.
+   *
+   * @param host ip address string
+   * @param port port for the resolver
+   */
+  public NativeCronvoyEngineBuilderImpl addCaresFallbackResolver(String host, int port) {
+    mCaresFallbackResolvers.add(new Pair<String, Integer>(host, port));
+    return this;
+  }
+
   /**
    * Set whether to map v4 address to v6.
    *
@@ -283,6 +296,6 @@ mUseGro, quicConnectionOptions(), quicClientConnectionOptions(), quicHints(),
         mH2ConnectionKeepaliveTimeoutSeconds, mMaxConnectionsPerHost, mStreamIdleTimeoutSeconds,
         mPerTryIdleTimeoutSeconds, mAppVersion, mAppId, mTrustChainVerification, nativeFilterChain,
         platformFilterChain, stringAccessors, keyValueStores, mRuntimeGuards,
-        mEnablePlatformCertificatesValidation, mUpstreamTlsSni);
+        mEnablePlatformCertificatesValidation, mUpstreamTlsSni, mCaresFallbackResolvers);
   }
 }

mobile/library/jni/jni_impl.cc

@@ -1230,6 +1230,7 @@ void configureBuilder(Envoy::JNI::JniHelper& jni_helper, jlong connect_timeout_s
                       jstring app_version, jstring app_id, jboolean trust_chain_verification,
                       jobjectArray filter_chain, jboolean enable_platform_certificates_validation,
                       jstring upstream_tls_sni, jobjectArray runtime_guards,
+                      jobjectArray cares_fallback_resolvers,
                       Envoy::Platform::EngineBuilder& builder) {
   builder.addConnectTimeoutSeconds((connect_timeout_seconds));
   builder.addDnsRefreshSeconds((dns_refresh_seconds));
@@ -1271,6 +1272,12 @@ void configureBuilder(Envoy::JNI::JniHelper& jni_helper, jlong connect_timeout_s
   }
   builder.enablePortMigration(enable_port_migration);
   builder.setUseCares(use_cares == JNI_TRUE);
+  if (use_cares == JNI_TRUE) {
+    auto resolvers = javaObjectArrayToStringPairVector(jni_helper, cares_fallback_resolvers);
+    for (const auto& [host, port] : resolvers) {
+      builder.addCaresFallbackResolver(host, stoi(port));
+    }
+  }
   builder.setUseGroIfAvailable(use_gro == JNI_TRUE);
   builder.enableInterfaceBinding(enable_interface_binding == JNI_TRUE);
   builder.enableDrainPostDnsRefresh(enable_drain_post_dns_refresh == JNI_TRUE);
@@ -1326,22 +1333,23 @@ extern "C" JNIEXPORT jlong JNICALL Java_io_envoyproxy_envoymobile_engine_JniLibr
     jlong stream_idle_timeout_seconds, jlong per_try_idle_timeout_seconds, jstring app_version,
     jstring app_id, jboolean trust_chain_verification, jobjectArray filter_chain,
     jboolean enable_platform_certificates_validation, jstring upstream_tls_sni,
-    jobjectArray runtime_guards) {
+    jobjectArray runtime_guards, jobjectArray cares_fallback_resolvers) {
   Envoy::JNI::JniHelper jni_helper(env);
   Envoy::Platform::EngineBuilder builder;
 
-  configureBuilder(
-      jni_helper, connect_timeout_seconds, dns_refresh_seconds, dns_failure_refresh_seconds_base,
-      dns_failure_refresh_seconds_max, dns_query_timeout_seconds, dns_min_refresh_seconds,
-      dns_preresolve_hostnames, enable_dns_cache, dns_cache_save_interval_seconds, dns_num_retries,
-      enable_drain_post_dns_refresh, enable_http3, use_cares, force_v6, use_gro,
-      http3_connection_options, http3_client_connection_options, quic_hints,
-      quic_canonical_suffixes, enable_gzip_decompression, enable_brotli_decompression,
-      enable_port_migration, enable_socket_tagging, enable_interface_binding,
-      h2_connection_keepalive_idle_interval_milliseconds, h2_connection_keepalive_timeout_seconds,
-      max_connections_per_host, stream_idle_timeout_seconds, per_try_idle_timeout_seconds,
-      app_version, app_id, trust_chain_verification, filter_chain,
-      enable_platform_certificates_validation, upstream_tls_sni, runtime_guards, builder);
+  configureBuilder(jni_helper, connect_timeout_seconds, dns_refresh_seconds,
+                   dns_failure_refresh_seconds_base, dns_failure_refresh_seconds_max,
+                   dns_query_timeout_seconds, dns_min_refresh_seconds, dns_preresolve_hostnames,
+                   enable_dns_cache, dns_cache_save_interval_seconds, dns_num_retries,
+                   enable_drain_post_dns_refresh, enable_http3, use_cares, force_v6, use_gro,
+                   http3_connection_options, http3_client_connection_options, quic_hints,
+                   quic_canonical_suffixes, enable_gzip_decompression, enable_brotli_decompression,
+                   enable_port_migration, enable_socket_tagging, enable_interface_binding,
+                   h2_connection_keepalive_idle_interval_milliseconds,
+                   h2_connection_keepalive_timeout_seconds, max_connections_per_host,
+                   stream_idle_timeout_seconds, per_try_idle_timeout_seconds, app_version, app_id,
+                   trust_chain_verification, filter_chain, enable_platform_certificates_validation,
+                   upstream_tls_sni, runtime_guards, cares_fallback_resolvers, builder);
 
   return reinterpret_cast<intptr_t>(builder.generateBootstrap().release());
 }

mobile/library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt

@@ -1,5 +1,6 @@
 package io.envoyproxy.envoymobile
 
+import android.util.Pair
 import io.envoyproxy.envoymobile.engine.EnvoyConfiguration
 import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification
 import io.envoyproxy.envoymobile.engine.EnvoyEngine
@@ -39,6 +40,7 @@ open class EngineBuilder() {
   private var enableDrainPostDnsRefresh = false
   internal var enableHttp3 = true
   internal var useCares = false
+  internal var caresFallbackResolvers = mutableListOf<Pair<String, Int>>()
   internal var forceV6 = true
   private var useGro = false
   private var http3ConnectionOptions = ""
@@ -219,6 +221,18 @@ open class EngineBuilder() {
     return this
   }
 
+  /**
+   * Add fallback resolver to c_ares.
+   *
+   * @param host ip address string
+   * @param port port for the resolver
+   * @return This builder.
+   */
+  fun addCaresFallbackResolver(host: String, port: Int): EngineBuilder {
+    this.caresFallbackResolvers.add(Pair(host, port))
+    return this
+  }
+
   /**
    * Specify whether local ipv4 addresses should be mapped to ipv6. Defaults to true.
    *
@@ -582,6 +596,7 @@ open class EngineBuilder() {
         runtimeGuards,
         enablePlatformCertificatesValidation,
         upstreamTlsSni,
+        caresFallbackResolvers,
       )
 
     return EngineImpl(engineType(), engineConfiguration, logLevel)

mobile/test/java/io/envoyproxy/envoymobile/engine/EnvoyConfigurationTest.kt

@@ -1,5 +1,6 @@
 package io.envoyproxy.envoymobile.engine
 
+import android.util.Pair
 import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilter
 import io.envoyproxy.envoymobile.engine.types.EnvoyHTTPFilterFactory
 import io.envoyproxy.envoymobile.engine.EnvoyConfiguration.TrustChainVerification
@@ -82,6 +83,7 @@ class EnvoyConfigurationTest {
     enableDrainPostDnsRefresh: Boolean = false,
     enableHttp3: Boolean = true,
     enableCares: Boolean = false,
+    caresFallbackResolvers: MutableList<Pair<String, Int>> = mutableListOf(Pair("1.2.3.4", 88)),
     forceV6: Boolean = true,
     enableGro: Boolean = false,
     http3ConnectionOptions: String = "5RTO",
@@ -156,6 +158,7 @@ class EnvoyConfigurationTest {
       runtimeGuards,
       enablePlatformCertificatesValidation,
       upstreamTlsSni,
+      caresFallbackResolvers,
     )
   }
 
@@ -261,6 +264,8 @@ class EnvoyConfigurationTest {
 
     // enableCares = true
     assertThat(resolvedTemplate).contains("envoy.network.dns_resolver.cares")
+    assertThat(resolvedTemplate).contains("address: \"1.2.3.4\"");
+    assertThat(resolvedTemplate).contains("port_value: 88");
 
     // enableGro = true
     assertThat(resolvedTemplate).contains("key: \"prefer_quic_client_udp_gro\" value { bool_value: true }")