[WIP] Sequence Numbers powered Compare and Set operations

docs/plugins/ingest-attachment.asciidoc

@@ -63,6 +63,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "attachment": {
@@ -74,7 +76,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 To specify only some fields to be extracted:
@@ -146,6 +148,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 35,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "attachment": {
@@ -157,7 +161,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 [source,js]
@@ -194,6 +198,8 @@ Returns this:
   "_type": "_doc",
   "_id": "my_id_2",
   "_version": 1,
+  "_seq_no": 40,
+  "_primary_term": 1,
   "_source": {
     "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=",
     "max_size": 5,
@@ -206,7 +212,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 [[ingest-attachment-with-arrays]]
@@ -285,6 +291,8 @@ Returns this:
   "_type" : "_doc",
   "_id" : "my_id",
   "_version" : 1,
+  "_seq_no" : 50,
+  "_primary_term" : 1,
   "found" : true,
   "_source" : {
     "attachments" : [
@@ -312,7 +320,7 @@ Returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Note that the `target_field` needs to be set, otherwise the

docs/plugins/ingest-geoip.asciidoc

@@ -75,6 +75,8 @@ Which returns:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 55,
+  "_primary_term": 1,
   "_source": {
     "ip": "8.8.8.8",
     "geoip": {
@@ -85,7 +87,7 @@ Which returns:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term":1/"_primary_term" : $body._primary_term/]
 
 Here is an example that uses the default country database and adds the
 geographical information to the `geo` field based on the `ip` field`. Note that
@@ -124,6 +126,8 @@ returns this:
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 65,
+  "_primary_term": 1,
   "_source": {
     "ip": "8.8.8.8",
     "geo": {
@@ -133,7 +137,7 @@ returns this:
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Not all IP addresses find geo information from the database, When this
@@ -174,13 +178,15 @@ Which returns:
   "_type" : "_doc",
   "_id" : "my_id",
   "_version" : 1,
+  "_seq_no" : 71,
+  "_primary_term": 1,
   "found" : true,
   "_source" : {
     "ip" : "80.231.5.0"
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 [[ingest-geoip-mappings-note]]
 ===== Recognizing Location as a Geopoint

docs/plugins/ingest-user-agent.asciidoc

@@ -57,6 +57,8 @@ Which returns
   "_type": "_doc",
   "_id": "my_id",
   "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
   "_source": {
     "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
     "user_agent": {
@@ -73,7 +75,7 @@ Which returns
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/]
 
 ===== Using a custom regex file
 To use a custom regex file for parsing the user agents, that file has to be put into the `config/ingest-user-agent` directory and

docs/reference/docs/bulk.asciidoc

@@ -38,7 +38,7 @@ optional_source\n
 
 *NOTE*: the final line of data must end with a newline character `\n`. Each newline character
 may be preceded by a carriage return `\r`. When sending requests to this endpoint the
-`Content-Type` header should be set to `application/x-ndjson`.
+`Content-Type` header should be set to `application/x-  ndjson`.
 
 The possible actions are `index`, `create`, `delete` and `update`.
 `index` and `create` expect a source on the next

docs/reference/docs/get.asciidoc

@@ -21,6 +21,8 @@ The result of the above get operation is:
     "_type" : "_doc",
     "_id" : "0",
     "_version" : 1,
+    "_seq_no" : 10,
+    "_primary_term" : 1,
     "found": true,
     "_source" : {
         "user" : "kimchy",
@@ -30,7 +32,7 @@ The result of the above get operation is:
     }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 The above result includes the `_index`, `_type`, `_id` and `_version`
 of the document we wish to retrieve, including the actual `_source`
@@ -156,6 +158,8 @@ The result of the above get operation is:
    "_type": "_doc",
    "_id": "1",
    "_version": 1,
+   "_seq_no" : 22,
+   "_primary_term" : 1,
    "found": true,
    "fields": {
       "tags": [
@@ -164,7 +168,7 @@ The result of the above get operation is:
    }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Field values fetched from the document itself are always returned as an array.
@@ -199,6 +203,8 @@ The result of the above get operation is:
    "_type": "_doc",
    "_id": "2",
    "_version": 1,
+   "_seq_no" : 13,
+   "_primary_term" : 1,
    "_routing": "user1",
    "found": true,
    "fields": {
@@ -208,7 +214,7 @@ The result of the above get operation is:
    }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 Also only leaf fields can be returned via the `stored_field` option. So object fields can't be returned and such requests
 will fail.

docs/reference/docs/reindex.asciidoc

@@ -870,13 +870,15 @@ which will return:
   "_index": "test2",
   "_type": "_doc",
   "_version": 1,
+  "_seq_no": 44,
+  "_primary_term": 1,
   "_source": {
     "text": "words words",
     "tag": "foo"
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/]
 
 [float]
 [[docs-reindex-slice]]

docs/reference/getting-started.asciidoc

@@ -421,7 +421,7 @@ And the response:
   "_primary_term" : 1
 }
 --------------------------------------------------
-// TESTRESPONSE[s/"_seq_no" : 0/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 From the above, we can see that a new customer document was successfully created inside the customer index. The document also has an internal id of 1 which we specified at index time.
 
@@ -445,11 +445,13 @@ And the response:
   "_type" : "_doc",
   "_id" : "1",
   "_version" : 1,
+  "_seq_no" : 25,
+  "_primary_term" : 1,
   "found" : true,
   "_source" : { "name": "John Doe" }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 Nothing out of the ordinary here other than a field, `found`, stating that we found a document with the requested ID 1 and another field, `_source`, which returns the full JSON document that we indexed from the previous step.
 

docs/reference/ingest/ingest-node.asciidoc

@@ -730,13 +730,15 @@ GET test/_doc/2
   "_type": "_doc",
   "_id": "2",
   "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
   "found": true,
   "_source": {
     "foo": "bar"
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term": 1/"_primary_term" : $body._primary_term/]
 ////
 
 The source document can also use dot delimited fields to represent nested fields.
@@ -967,6 +969,8 @@ GET test/_doc/2
   "_type": "_doc",
   "_id": "2",
   "_version": 1,
+  "_seq_no": 34,
+  "_primary_term": 1,
   "found": true,
   "_source": {
     "tags": [
@@ -976,7 +980,7 @@ GET test/_doc/2
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 ////
 
 
@@ -1088,6 +1092,8 @@ GET test/_doc/1
   "_type": "_doc",
   "_id": "1",
   "_version": 1,
+  "_seq_no": 60,
+  "_primary_term": 1,
   "found": true,
   "_source": {
     "href": {
@@ -1097,7 +1103,7 @@ GET test/_doc/1
   }
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 Regular expressions can be expensive and should be avoided if viable
@@ -1548,11 +1554,11 @@ PUT /myindex/_doc/1?pipeline=monthlyindex
     "successful" : 1,
     "failed" : 0
   },
-  "_seq_no" : 0,
+  "_seq_no" : 55,
   "_primary_term" : 1
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 
 The above request will not index this document into the `myindex` index, but into the `myindex-2016-04-01` index because
@@ -2787,11 +2793,11 @@ Response from the index request:
     "successful": 1,
     "failed": 0
   },
-  "_seq_no": 0,
+  "_seq_no": 66,
   "_primary_term": 1,
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 Indexed document:
 
@@ -2963,11 +2969,11 @@ The response from the above index request:
     "successful": 1,
     "failed": 0
   },
-  "_seq_no": 0,
+  "_seq_no": 89,
   "_primary_term": 1,
 }
 --------------------------------------------------
-// TESTRESPONSE
+// TESTRESPONSE[s/"_seq_no": \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
 
 In the above response, you can see that our document was actually indexed into `my_index` instead of
 `any_index`. This type of manipulation is often convenient in pipelines that have various branches of transformation,

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolateQueryBuilderTests.java

@@ -54,6 +54,7 @@
 import java.util.Map;
 import java.util.Set;
 
+import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO;
 import static org.hamcrest.Matchers.equalTo;
 
 public class PercolateQueryBuilderTests extends AbstractQueryTestCase<PercolateQueryBuilder> {
@@ -152,12 +153,13 @@ protected GetResponse executeGet(GetRequest getRequest) {
         assertThat(getRequest.version(), Matchers.equalTo(indexedDocumentVersion));
         if (indexedDocumentExists) {
             return new GetResponse(
-                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, 0L, true,
+                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, 0, 1, 0L, true,
                             documentSource.iterator().next(), Collections.emptyMap())
             );
         } else {
             return new GetResponse(
-                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, -1, false, null, Collections.emptyMap())
+                    new GetResult(indexedDocumentIndex, indexedDocumentType, indexedDocumentId, UNASSIGNED_SEQ_NO, 0, -1,
+                        false, null, Collections.emptyMap())
             );
         }
     }

rest-api-spec/src/main/resources/rest-api-spec/api/delete.json

@@ -43,6 +43,14 @@
           "type" : "time",
           "description" : "Explicit operation timeout"
         },
+        "if_seq_no_match" : {
+          "type" : "number",
+          "description" : "only perform the delete operation if the last operation to have changed the document has the specified sequence number"
+        },
+        "if_primary_term_match" : {
+          "type" : "number",
+          "description" : "only perform the delete operation if the last operation to have changed the document has the specified primary term"
+        },
         "version" : {
           "type" : "number",
           "description" : "Explicit version number for concurrency control"

rest-api-spec/src/main/resources/rest-api-spec/api/index.json

@@ -57,6 +57,14 @@
           "options" : ["internal", "external", "external_gte", "force"],
           "description" : "Specific version type"
         },
+        "if_seq_no_match" : {
+          "type" : "number",
+          "description" : "only perform the index operation if the last operation to have changed the document has the specified sequence number"
+        },
+        "if_primary_term_match" : {
+          "type" : "number",
+          "description" : "only perform the index operation if the last operation to have changed the document has the specified primary term"
+        },
         "pipeline" : {
           "type" : "string",
           "description" : "The pipeline id to preprocess incoming documents with"