Require object_type for object fields (#628)

A field of type object without object_type probably assumes that the type should be keyword,
but Fleet is producing invalid mappings (see #624). So better to disallow these mappings to
remove ambiguities.

code/go/pkg/validator/validator_test.go

@@ -136,6 +136,7 @@ func TestValidateFile(t *testing.T) {
 			[]string{
 				`field 0.type: 0.type must be one of the following: "aggregate_metric_double", "alias", "histogram", "constant_keyword", "text", "match_only_text", "keyword", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "date", "date_nanos", "boolean", "binary", "integer_range", "float_range", "long_range", "double_range", "date_range", "ip_range", "group", "geo_point", "object", "ip", "nested", "flattened", "wildcard", "version", "unsigned_long"`,
 				`field "my_custom_date" of type keyword can't set date_format. date_format is allowed for date field type only`,
+				`field 2: object_type is required`,
 			},
 		},
 		"deploy_custom_agent_invalid_property": {

spec/changelog.yml

@@ -28,6 +28,9 @@
   - description: Make exception for legacy input controls
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/619
+  - description: object_type is mandatory when object type is used
+    type: bugfix
+    link: https://github.com/elastic/package-spec/pull/628
 - version: 2.13.1-next
   changes:
   - description: Prepare for next version

spec/integration/data_stream/fields/fields.spec.yml

@@ -46,11 +46,14 @@ spec:
         description: >
           Name of field. Names containing dots are automatically split into
           sub-fields.
+          Names with wildcards generate dynamic mappings.
         type: string
         pattern: '^[\-*_\/@A-Za-z0-9]+(\.[\-*_\/@A-Za-z0-9]+)*$'
 
       type:
-        description: Datatype of field
+        description: >
+          Datatype of field. If the type is set to object, a dynamic mapping is created. In this case, if the name doesn't
+          contain any wildcard, the wildcard is added as the last segment of the path.
         type: string
         enum:
         - aggregate_metric_double
@@ -521,11 +524,25 @@ spec:
           required:
             - type
             - object_type
+      - if:
+          properties:
+            type:
+              const: object
+          required:
+            - type
+        then:
+          required:
+            - object_type
+
     required:
     - name
 
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 3.0.0
+    patch:
+      - op: remove
+        path: "/items/allOf/6" # removing requirement of object_type when type is object
   - before: 2.10.0
     patch:
       - op: remove

test/packages/bad_fields/data_stream/foo/fields/fields.yml

@@ -2,4 +2,6 @@
   type: array
 - name: my_custom_date
   type: keyword
-  date_format: yyyy-MM-dd
+  date_format: yyyy-MM-dd
+- name: object_without_object_type.*
+  type: object

test/packages/bad_fields/manifest.yml

@@ -1,4 +1,4 @@
-format_version: 2.7.0
+format_version: 3.0.0
 name: bad_fields
 title: "Bad Fields"
 version: 0.0.1
@@ -9,8 +9,10 @@ type: integration
 categories:
   - custom
 conditions:
-  kibana.version: "^8.4.1"
-  elastic.subscription: "basic"
+  kibana:
+    version: "^8.4.1"
+  elastic:
+    subscription: "basic"
 screenshots:
   - src: /img/sample-screenshot.png
     title: Sample screenshot
@@ -30,4 +32,5 @@ policy_templates:
         title: Collect sample logs from instances
         description: Collecting sample logs
 owner:
+  type: elastic
   github: elastic/ecosystem

test/packages/good_v3/elasticsearch/transform/metadata_united/fields/fields.yml

@@ -356,4 +356,5 @@
           type: date
         - name: user_provided_metadata
           type: object
+          object_type: keyword
           enabled: false