Detect zero receiver of modInverse (may not converge and time out

in that case).

See #23502

Review URL: https://codereview.chromium.org//1177063002.

runtime/lib/bigint.dart

@@ -1542,6 +1542,7 @@ class _Bigint extends _IntegerImplementation implements int {
   int modInverse(int m) {
     if (m is! int) throw new ArgumentError(m);
     if (m <= 0) throw new RangeError(m);
+    if (_used == 0) return 0;
     m = m._toBigint();
     // TODO(regis): Implement modInverse for an even modulus.
     if (m.isEven) throw new UnimplementedError();

runtime/lib/integers.dart

@@ -297,6 +297,7 @@ class _IntegerImplementation extends _Num {
     if (m is _Bigint) {
       return _toBigint().modInverse(m);
     }
+    if (this == 0) return 0;
     bool ac = m.isEven;
     int u = m;
     int v = this;

sdk/lib/_internal/compiler/js_lib/js_number.dart

@@ -415,6 +415,7 @@ class JSInt extends JSNumber implements int, double {
   int modInverse(int m) {
     if (m is! int) throw new ArgumentError(m);
     if (m <= 0) throw new RangeError(m);
+    if (this == 0) return 0;
     bool ac = m.isEven;
     int u = m;
     int v = this;

sdk/lib/core/int.dart

@@ -116,6 +116,7 @@ abstract class int extends num {
    * modulo [modulus].
    *
    * The [modulus] must be positive.
+   * Returns 0 if no modular inverse exists.
    */
   int modInverse(int modulus);
 

tests/corelib/big_integer_arith_vm_test.dart

@@ -235,6 +235,9 @@ UnimplementedEvenModulusModInverse(x, m) {
 
 testBigintModInverse() {
   var x, m;
+  x = 0;
+  m = 1000000001;
+  Expect.equals(0, x.modInverse(m));  // Not coprime.
   x = 1234567890;
   m = 19;
   Expect.equals(11, x.modInverse(m));