diff --git a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs index fd2ebddba..03075d7ca 100644 --- a/crates/iceberg/src/expr/visitors/manifest_evaluator.rs +++ b/crates/iceberg/src/expr/visitors/manifest_evaluator.rs @@ -293,9 +293,9 @@ mod test { sequence_number: 0, min_sequence_number: 0, added_snapshot_id: 0, - added_data_files_count: None, - existing_data_files_count: None, - deleted_data_files_count: None, + added_files_count: None, + existing_files_count: None, + deleted_files_count: None, added_rows_count: None, existing_rows_count: None, deleted_rows_count: None, diff --git a/crates/iceberg/src/spec/manifest.rs b/crates/iceberg/src/spec/manifest.rs index b1eb21653..f61c0acaa 100644 --- a/crates/iceberg/src/spec/manifest.rs +++ b/crates/iceberg/src/spec/manifest.rs @@ -306,9 +306,9 @@ impl ManifestWriter { sequence_number: UNASSIGNED_SEQUENCE_NUMBER, min_sequence_number: self.min_seq_num.unwrap_or(UNASSIGNED_SEQUENCE_NUMBER), added_snapshot_id: self.snapshot_id, - added_data_files_count: Some(self.added_files), - existing_data_files_count: Some(self.existing_files), - deleted_data_files_count: Some(self.deleted_files), + added_files_count: Some(self.added_files), + existing_files_count: Some(self.existing_files), + deleted_files_count: Some(self.deleted_files), added_rows_count: Some(self.added_rows), existing_rows_count: Some(self.existing_rows), deleted_rows_count: Some(self.deleted_rows), diff --git a/crates/iceberg/src/spec/manifest_list.rs b/crates/iceberg/src/spec/manifest_list.rs index 26a4acc60..2bb5e309e 100644 --- a/crates/iceberg/src/spec/manifest_list.rs +++ b/crates/iceberg/src/spec/manifest_list.rs @@ -297,7 +297,7 @@ mod _const_schema { Lazy::new(|| { Arc::new(NestedField::required( 504, - "added_data_files_count", + "added_files_count", Type::Primitive(PrimitiveType::Int), )) }) @@ -315,7 +315,7 @@ mod _const_schema { Lazy::new(|| { Arc::new(NestedField::required( 505, - "existing_data_files_count", + "existing_files_count", Type::Primitive(PrimitiveType::Int), )) }) @@ -333,7 +333,7 @@ mod _const_schema { Lazy::new(|| { Arc::new(NestedField::required( 506, - "deleted_data_files_count", + "deleted_files_count", Type::Primitive(PrimitiveType::Int), )) }) @@ -537,17 +537,17 @@ pub struct ManifestFile { /// /// Number of entries in the manifest that have status ADDED, when null /// this is assumed to be non-zero - pub added_data_files_count: Option, + pub added_files_count: Option, /// field: 505 /// /// Number of entries in the manifest that have status EXISTING (0), /// when null this is assumed to be non-zero - pub existing_data_files_count: Option, + pub existing_files_count: Option, /// field: 506 /// /// Number of entries in the manifest that have status DELETED (2), /// when null this is assumed to be non-zero - pub deleted_data_files_count: Option, + pub deleted_files_count: Option, /// field: 512 /// /// Number of rows in all of files in the manifest that have status @@ -809,11 +809,11 @@ pub(super) mod _serde { pub min_sequence_number: i64, pub added_snapshot_id: i64, #[serde(alias = "added_data_files_count", alias = "added_files_count")] - pub added_data_files_count: i32, + pub added_files_count: i32, #[serde(alias = "existing_data_files_count", alias = "existing_files_count")] - pub existing_data_files_count: i32, + pub existing_files_count: i32, #[serde(alias = "deleted_data_files_count", alias = "deleted_files_count")] - pub deleted_data_files_count: i32, + pub deleted_files_count: i32, pub added_rows_count: i64, pub existing_rows_count: i64, pub deleted_rows_count: i64, @@ -895,9 +895,9 @@ pub(super) mod _serde { sequence_number: self.sequence_number, min_sequence_number: self.min_sequence_number, added_snapshot_id: self.added_snapshot_id, - added_data_files_count: Some(self.added_data_files_count.try_into()?), - existing_data_files_count: Some(self.existing_data_files_count.try_into()?), - deleted_data_files_count: Some(self.deleted_data_files_count.try_into()?), + added_files_count: Some(self.added_files_count.try_into()?), + existing_files_count: Some(self.existing_files_count.try_into()?), + deleted_files_count: Some(self.deleted_files_count.try_into()?), added_rows_count: Some(self.added_rows_count.try_into()?), existing_rows_count: Some(self.existing_rows_count.try_into()?), deleted_rows_count: Some(self.deleted_rows_count.try_into()?), @@ -917,15 +917,15 @@ pub(super) mod _serde { manifest_length: self.manifest_length, partition_spec_id: self.partition_spec_id, added_snapshot_id: self.added_snapshot_id, - added_data_files_count: self + added_files_count: self .added_data_files_count .map(TryInto::try_into) .transpose()?, - existing_data_files_count: self + existing_files_count: self .existing_data_files_count .map(TryInto::try_into) .transpose()?, - deleted_data_files_count: self + deleted_files_count: self .deleted_data_files_count .map(TryInto::try_into) .transpose()?, @@ -988,8 +988,8 @@ pub(super) mod _serde { sequence_number: value.sequence_number, min_sequence_number: value.min_sequence_number, added_snapshot_id: value.added_snapshot_id, - added_data_files_count: value - .added_data_files_count + added_files_count: value + .added_files_count .ok_or_else(|| { Error::new( crate::ErrorKind::DataInvalid, @@ -997,8 +997,8 @@ pub(super) mod _serde { ) })? .try_into()?, - existing_data_files_count: value - .existing_data_files_count + existing_files_count: value + .existing_files_count .ok_or_else(|| { Error::new( crate::ErrorKind::DataInvalid, @@ -1006,8 +1006,8 @@ pub(super) mod _serde { ) })? .try_into()?, - deleted_data_files_count: value - .deleted_data_files_count + deleted_files_count: value + .deleted_files_count .ok_or_else(|| { Error::new( crate::ErrorKind::DataInvalid, @@ -1060,15 +1060,15 @@ pub(super) mod _serde { partition_spec_id: value.partition_spec_id, added_snapshot_id: value.added_snapshot_id, added_data_files_count: value - .added_data_files_count + .added_files_count .map(TryInto::try_into) .transpose()?, existing_data_files_count: value - .existing_data_files_count + .existing_files_count .map(TryInto::try_into) .transpose()?, deleted_data_files_count: value - .deleted_data_files_count + .deleted_files_count .map(TryInto::try_into) .transpose()?, added_rows_count: value.added_rows_count.map(TryInto::try_into).transpose()?, @@ -1116,9 +1116,9 @@ mod test { sequence_number: 0, min_sequence_number: 0, added_snapshot_id: 1646658105718557341, - added_data_files_count: Some(3), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1166,9 +1166,9 @@ mod test { sequence_number: 1, min_sequence_number: 1, added_snapshot_id: 377075049360453639, - added_data_files_count: Some(1), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1183,9 +1183,9 @@ mod test { sequence_number: 1, min_sequence_number: 1, added_snapshot_id: 377075049360453639, - added_data_files_count: Some(1), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1254,9 +1254,9 @@ mod test { sequence_number: 0, min_sequence_number: 0, added_snapshot_id: 1646658105718557341, - added_data_files_count: Some(3), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1282,9 +1282,9 @@ mod test { sequence_number: 1, min_sequence_number: 1, added_snapshot_id: 377075049360453639, - added_data_files_count: Some(1), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1295,7 +1295,7 @@ mod test { let result = serde_json::to_string(&manifest_list).unwrap(); assert_eq!( result, - r#"[{"manifest_path":"s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro","manifest_length":6926,"partition_spec_id":1,"content":0,"sequence_number":1,"min_sequence_number":1,"added_snapshot_id":377075049360453639,"added_data_files_count":1,"existing_data_files_count":0,"deleted_data_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":[{"contains_null":false,"contains_nan":false,"lower_bound":[1,0,0,0,0,0,0,0],"upper_bound":[1,0,0,0,0,0,0,0]}],"key_metadata":null}]"# + r#"[{"manifest_path":"s3a://icebergdata/demo/s1/t1/metadata/05ffe08b-810f-49b3-a8f4-e88fc99b254a-m0.avro","manifest_length":6926,"partition_spec_id":1,"content":0,"sequence_number":1,"min_sequence_number":1,"added_snapshot_id":377075049360453639,"added_files_count":1,"existing_files_count":0,"deleted_files_count":0,"added_rows_count":3,"existing_rows_count":0,"deleted_rows_count":0,"partitions":[{"contains_null":false,"contains_nan":false,"lower_bound":[1,0,0,0,0,0,0,0],"upper_bound":[1,0,0,0,0,0,0,0]}],"key_metadata":null}]"# ); } @@ -1310,9 +1310,9 @@ mod test { sequence_number: 0, min_sequence_number: 0, added_snapshot_id: 1646658105718557341, - added_data_files_count: Some(3), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1366,9 +1366,9 @@ mod test { sequence_number: UNASSIGNED_SEQUENCE_NUMBER, min_sequence_number: UNASSIGNED_SEQUENCE_NUMBER, added_snapshot_id: snapshot_id, - added_data_files_count: Some(1), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(1), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0), @@ -1420,9 +1420,9 @@ mod test { sequence_number: 0, min_sequence_number: 0, added_snapshot_id: 1646658105718557341, - added_data_files_count: Some(3), - existing_data_files_count: Some(0), - deleted_data_files_count: Some(0), + added_files_count: Some(3), + existing_files_count: Some(0), + deleted_files_count: Some(0), added_rows_count: Some(3), existing_rows_count: Some(0), deleted_rows_count: Some(0),