Core: Added _shards header to all write responses.

The header indicates to how many shard copies (primary and replicas shards) a write was supposed to go to, to how many
shard copies to write succeeded and potentially captures shard failures if writing into a replica shard fails.

For async writes it also includes the number of shards a write is still pending.

Closes #7994

docs/reference/docs/delete-by-query.asciidoc

@@ -32,9 +32,9 @@ commands is:
     "_indices" : {
         "twitter" : { 
             "_shards" : {
-                "total" : 5,
-                "successful" : 5,
-                "failed" : 0
+                "total" : 10,
+                "failed" : 0,
+                "successful" : 10,
             }
         }
     }

docs/reference/docs/delete.asciidoc

@@ -16,6 +16,11 @@ The result of the above delete operation is:
 [source,js]
 --------------------------------------------------
 {
+    "_shards" : {
+        "total" : 10,
+        "failed" : 0,
+        "successful" : 10
+    },
     "found" : true,
     "_index" : "twitter",
     "_type" : "tweet",

docs/reference/docs/index_.asciidoc

@@ -19,6 +19,11 @@ The result of the above index operation is:
 [source,js]
 --------------------------------------------------
 {
+    "_shards" : {
+        "total" : 10,
+        "failed" : 0,
+        "successful" : 10
+    },
     "_index" : "twitter",
     "_type" : "tweet",
     "_id" : "1",
@@ -27,6 +32,20 @@ The result of the above index operation is:
 }
 --------------------------------------------------
 
+The `_shards` header provides information about the replication process of the index operation.
+* `total` - Indicates to how many shard copies (primary and replica shards) the index operation should be executed on.
+* `successful`- Indicates the number of shard copies the index operation succeeded on.
+* `pending` - Indicates how many shard copies this index operation still needs to go to at the time index operation
+ succeeded on the primary shard. This field is only returned if `async` replication is used.
+* `failures` - An array that contains replication related errors in the case an index operation failed on a replica shard.
+
+The index operation is successful in the case `successful` is at least 1.
+
+NOTE:   Replica shards may not all be started when an indexing operation successfully returns (by default, a  quorum is
+        required). In that case, `total` will be equal to the total shards based on the index replica settings and
+        `successful` will be equal to the number of shard started (primary plus replicas). As there were no failures,
+        the `failed` will be 0.
+
 [float]
 [[index-creation]]
 === Automatic Index Creation

docs/reference/migration/migrate_2_0.asciidoc

@@ -114,4 +114,12 @@ well.
 
 The `parent` parameter has been removed from the update request. Before 2.x it just set the routing parameter. The
 `routing` setting should be used instead. The `parent` setting was confusing, because it had the impression that the parent
-a child documents points to can be changed but this is not true.
+a child documents points to can be changed but this is not true.
+
+==== Delete by query
+
+The meaning of the `_shards` headers in the delete by query response has changed. Before version 2.0 the `total`,
+`successful` and `failed` fields in the header are based on the number of primary shards. The failures on replica
+shards aren't being kept track of. From version 2.0 the stats in the `_shards` header are based on all shards
+of an index. The http status code is left unchanged and is only based on failures that occurred while executing on
+primary shards.

rest-api-spec/test/delete/11_shard_header.yaml

@@ -0,0 +1,36 @@
+---
+"Delete check shard header":
+
+ - do:
+      indices.create:
+        index: foobar
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+ - do:
+      cluster.health:
+          wait_for_status: green
+
+ - do:
+      index:
+          index:  foobar
+          type:   baz
+          id:     1
+          body:   { foo: bar }
+
+ - do:
+       delete:
+           index:  foobar
+           type:   baz
+           id:     1
+
+ - match:   { _index:   foobar }
+ - match:   { _type:    baz }
+ - match:   { _id:      "1"}
+ - match:   { _version: 2}
+ - match:   { _shards.total: 1}
+ - match:   { _shards.successful: 1}
+ - match:   { _shards.failed: 0}
+ - is_false:   _shards.pending

rest-api-spec/test/index/11_shard_header.yaml

@@ -0,0 +1,56 @@
+---
+"Index check shard header":
+
+ - do:
+      indices.create:
+        index: foobar1
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+
+ - do:
+      indices.create:
+        index: foobar2
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "1"
+
+ - do:
+      cluster.health:
+          wait_for_status: green
+
+ - do:
+      index:
+          index:  foobar1
+          type:   baz
+          id:     1
+          body:   { foo: bar }
+
+ - match:   { _index:   foobar1 }
+ - match:   { _type:    baz }
+ - match:   { _id:      "1"}
+ - match:   { _version: 1}
+ - match:   { _shards.total: 1}
+ - match:   { _shards.successful: 1}
+ - match:   { _shards.failed: 0}
+ - is_false:   _shards.pending
+
+ - do:
+      index:
+          index:  foobar2
+          type:   baz
+          id:     1
+          replication:  async
+          body:   { foo: bar }
+
+ - match:   { _index:   foobar2 }
+ - match:   { _type:    baz }
+ - match:   { _id:      "1"}
+ - match:   { _version: 1}
+ - match:   { _shards.total: 2}
+ - match:   { _shards.successful: 1}
+ - match:   { _shards.failed: 0}
+ - match:   { _shards.pending: 1}

rest-api-spec/test/update/11_shard_header.yaml

@@ -0,0 +1,39 @@
+---
+"Update check shard header":
+
+ - do:
+      indices.create:
+        index: foobar
+        body:
+          settings:
+            number_of_shards: "1"
+            number_of_replicas: "0"
+
+ - do:
+      cluster.health:
+          wait_for_status: green
+
+ - do:
+      index:
+          index:  foobar
+          type:   baz
+          id:     1
+          body:   { foo: bar }
+
+ - do:
+      update:
+           index:  foobar
+           type:   baz
+           id:     1
+           body:
+               doc:
+                   foo: baz
+
+ - match:   { _index:   foobar }
+ - match:   { _type:    baz }
+ - match:   { _id:      "1"}
+ - match:   { _version: 2}
+ - match:   { _shards.total: 1}
+ - match:   { _shards.successful: 1}
+ - match:   { _shards.failed: 0}
+ - is_false:   _shards.pending