From 4e55aa4d2fb3b54c5049f97cb461b906efb488b7 Mon Sep 17 00:00:00 2001 From: Andrew Dye Date: Fri, 29 Sep 2023 16:15:52 -0700 Subject: [PATCH] Add driver and executor pod template fields to SparkJob Signed-off-by: Andrew Dye --- gen/pb-cpp/flyteidl/plugins/spark.pb.cc | 445 +++++++- gen/pb-cpp/flyteidl/plugins/spark.pb.h | 275 +++++ gen/pb-go/flyteidl/plugins/spark.pb.go | 166 ++- .../flyteidl/plugins/spark.pb.validate.go | 36 + gen/pb-java/flyteidl/plugins/Spark.java | 1013 ++++++++++++++++- gen/pb_python/flyteidl/plugins/spark_pb2.py | 23 +- gen/pb_python/flyteidl/plugins/spark_pb2.pyi | 13 +- gen/pb_rust/flyteidl.plugins.rs | 39 + protos/flyteidl/plugins/spark.proto | 24 +- 9 files changed, 1945 insertions(+), 89 deletions(-) diff --git a/gen/pb-cpp/flyteidl/plugins/spark.pb.cc b/gen/pb-cpp/flyteidl/plugins/spark.pb.cc index 60f8d21c9..d1f108328 100644 --- a/gen/pb-cpp/flyteidl/plugins/spark.pb.cc +++ b/gen/pb-cpp/flyteidl/plugins/spark.pb.cc @@ -16,6 +16,7 @@ // @@protoc_insertion_point(includes) #include +extern PROTOBUF_INTERNAL_EXPORT_flyteidl_2fcore_2ftasks_2eproto ::google::protobuf::internal::SCCInfo<3> scc_info_K8sPod_flyteidl_2fcore_2ftasks_2eproto; extern PROTOBUF_INTERNAL_EXPORT_flyteidl_2fplugins_2fspark_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_SparkJob_HadoopConfEntry_DoNotUse_flyteidl_2fplugins_2fspark_2eproto; extern PROTOBUF_INTERNAL_EXPORT_flyteidl_2fplugins_2fspark_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_SparkJob_SparkConfEntry_DoNotUse_flyteidl_2fplugins_2fspark_2eproto; extern PROTOBUF_INTERNAL_EXPORT_google_2fprotobuf_2fstruct_2eproto ::google::protobuf::internal::SCCInfo<0> scc_info_ListValue_google_2fprotobuf_2fstruct_2eproto; @@ -36,6 +37,8 @@ class SparkJob_HadoopConfEntry_DoNotUseDefaultTypeInternal { class SparkJobDefaultTypeInternal { public: ::google::protobuf::internal::ExplicitlyConstructed _instance; + const ::flyteidl::core::K8sPod* driverpod_; + const ::flyteidl::core::K8sPod* executorpod_; } _SparkJob_default_instance_; } // namespace plugins } // namespace flyteidl @@ -90,11 +93,12 @@ static void InitDefaultsSparkJob_flyteidl_2fplugins_2fspark_2eproto() { ::flyteidl::plugins::SparkJob::InitAsDefaultInstance(); } -::google::protobuf::internal::SCCInfo<3> scc_info_SparkJob_flyteidl_2fplugins_2fspark_2eproto = - {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 3, InitDefaultsSparkJob_flyteidl_2fplugins_2fspark_2eproto}, { +::google::protobuf::internal::SCCInfo<4> scc_info_SparkJob_flyteidl_2fplugins_2fspark_2eproto = + {{ATOMIC_VAR_INIT(::google::protobuf::internal::SCCInfoBase::kUninitialized), 4, InitDefaultsSparkJob_flyteidl_2fplugins_2fspark_2eproto}, { &scc_info_SparkJob_SparkConfEntry_DoNotUse_flyteidl_2fplugins_2fspark_2eproto.base, &scc_info_SparkJob_HadoopConfEntry_DoNotUse_flyteidl_2fplugins_2fspark_2eproto.base, - &scc_info_ListValue_google_2fprotobuf_2fstruct_2eproto.base,}}; + &scc_info_ListValue_google_2fprotobuf_2fstruct_2eproto.base, + &scc_info_K8sPod_flyteidl_2fcore_2ftasks_2eproto.base,}}; void InitDefaults_flyteidl_2fplugins_2fspark_2eproto() { ::google::protobuf::internal::InitSCC(&scc_info_SparkApplication_flyteidl_2fplugins_2fspark_2eproto.base); @@ -134,7 +138,7 @@ const ::google::protobuf::uint32 TableStruct_flyteidl_2fplugins_2fspark_2eproto: ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, _internal_metadata_), ~0u, // no _extensions_ - ~0u, // no _oneof_case_ + PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, _oneof_case_[0]), ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, applicationtype_), PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, mainapplicationfile_), @@ -145,6 +149,12 @@ const ::google::protobuf::uint32 TableStruct_flyteidl_2fplugins_2fspark_2eproto: PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, databricksconf_), PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, databrickstoken_), PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, databricksinstance_), + offsetof(::flyteidl::plugins::SparkJobDefaultTypeInternal, driverpod_), + PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, driverpodtemplatename_), + offsetof(::flyteidl::plugins::SparkJobDefaultTypeInternal, executorpod_), + PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, executorpodtemplatename_), + PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, driverPodValue_), + PROTOBUF_FIELD_OFFSET(::flyteidl::plugins::SparkJob, executorPodValue_), }; static const ::google::protobuf::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::flyteidl::plugins::SparkApplication)}, @@ -168,36 +178,42 @@ ::google::protobuf::internal::AssignDescriptorsTable assign_descriptors_table_fl const char descriptor_table_protodef_flyteidl_2fplugins_2fspark_2eproto[] = "\n\034flyteidl/plugins/spark.proto\022\020flyteidl" - ".plugins\032\034google/protobuf/struct.proto\"B" - "\n\020SparkApplication\".\n\004Type\022\n\n\006PYTHON\020\000\022\010" - "\n\004JAVA\020\001\022\t\n\005SCALA\020\002\022\005\n\001R\020\003\"\333\003\n\010SparkJob\022" - "@\n\017applicationType\030\001 \001(\0162\'.flyteidl.plug" - "ins.SparkApplication.Type\022\033\n\023mainApplica" - "tionFile\030\002 \001(\t\022\021\n\tmainClass\030\003 \001(\t\022<\n\tspa" - "rkConf\030\004 \003(\0132).flyteidl.plugins.SparkJob" - ".SparkConfEntry\022>\n\nhadoopConf\030\005 \003(\0132*.fl" - "yteidl.plugins.SparkJob.HadoopConfEntry\022" - "\024\n\014executorPath\030\006 \001(\t\022/\n\016databricksConf\030" - "\007 \001(\0132\027.google.protobuf.Struct\022\027\n\017databr" - "icksToken\030\010 \001(\t\022\032\n\022databricksInstance\030\t " - "\001(\t\0320\n\016SparkConfEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005va" - "lue\030\002 \001(\t:\0028\001\0321\n\017HadoopConfEntry\022\013\n\003key\030" - "\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001B9Z7github.com/f" - "lyteorg/flyteidl/gen/pb-go/flyteidl/plug" - "insb\006proto3" + ".plugins\032\031flyteidl/core/tasks.proto\032\034goo" + "gle/protobuf/struct.proto\"B\n\020SparkApplic" + "ation\".\n\004Type\022\n\n\006PYTHON\020\000\022\010\n\004JAVA\020\001\022\t\n\005S" + "CALA\020\002\022\005\n\001R\020\003\"\233\005\n\010SparkJob\022@\n\017applicatio" + "nType\030\001 \001(\0162\'.flyteidl.plugins.SparkAppl" + "ication.Type\022\033\n\023mainApplicationFile\030\002 \001(" + "\t\022\021\n\tmainClass\030\003 \001(\t\022<\n\tsparkConf\030\004 \003(\0132" + ").flyteidl.plugins.SparkJob.SparkConfEnt" + "ry\022>\n\nhadoopConf\030\005 \003(\0132*.flyteidl.plugin" + "s.SparkJob.HadoopConfEntry\022\024\n\014executorPa" + "th\030\006 \001(\t\022/\n\016databricksConf\030\007 \001(\0132\027.googl" + "e.protobuf.Struct\022\027\n\017databricksToken\030\010 \001" + "(\t\022\032\n\022databricksInstance\030\t \001(\t\022*\n\tdriver" + "Pod\030\n \001(\0132\025.flyteidl.core.K8sPodH\000\022\035\n\025dr" + "iverPodTemplateName\030\013 \001(\t\022,\n\013executorPod" + "\030\014 \001(\0132\025.flyteidl.core.K8sPodH\001\022\037\n\027execu" + "torPodTemplateName\030\r \001(\t\0320\n\016SparkConfEnt" + "ry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\0321\n\017Ha" + "doopConfEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(" + "\t:\0028\001B\020\n\016driverPodValueB\022\n\020executorPodVa" + "lueB9Z7github.com/flyteorg/flyteidl/gen/" + "pb-go/flyteidl/pluginsb\006proto3" ; ::google::protobuf::internal::DescriptorTable descriptor_table_flyteidl_2fplugins_2fspark_2eproto = { false, InitDefaults_flyteidl_2fplugins_2fspark_2eproto, descriptor_table_protodef_flyteidl_2fplugins_2fspark_2eproto, - "flyteidl/plugins/spark.proto", &assign_descriptors_table_flyteidl_2fplugins_2fspark_2eproto, 691, + "flyteidl/plugins/spark.proto", &assign_descriptors_table_flyteidl_2fplugins_2fspark_2eproto, 910, }; void AddDescriptors_flyteidl_2fplugins_2fspark_2eproto() { - static constexpr ::google::protobuf::internal::InitFunc deps[1] = + static constexpr ::google::protobuf::internal::InitFunc deps[2] = { + ::AddDescriptors_flyteidl_2fcore_2ftasks_2eproto, ::AddDescriptors_google_2fprotobuf_2fstruct_2eproto, }; - ::google::protobuf::internal::AddDescriptors(&descriptor_table_flyteidl_2fplugins_2fspark_2eproto, deps, 1); + ::google::protobuf::internal::AddDescriptors(&descriptor_table_flyteidl_2fplugins_2fspark_2eproto, deps, 2); } // Force running AddDescriptors() at dynamic initialization time. @@ -526,22 +542,76 @@ bool SparkJob_HadoopConfEntry_DoNotUse::_ParseMap(const char* begin, const char* void SparkJob::InitAsDefaultInstance() { ::flyteidl::plugins::_SparkJob_default_instance_._instance.get_mutable()->databricksconf_ = const_cast< ::google::protobuf::Struct*>( ::google::protobuf::Struct::internal_default_instance()); + ::flyteidl::plugins::_SparkJob_default_instance_.driverpod_ = const_cast< ::flyteidl::core::K8sPod*>( + ::flyteidl::core::K8sPod::internal_default_instance()); + ::flyteidl::plugins::_SparkJob_default_instance_.executorpod_ = const_cast< ::flyteidl::core::K8sPod*>( + ::flyteidl::core::K8sPod::internal_default_instance()); } class SparkJob::HasBitSetters { public: static const ::google::protobuf::Struct& databricksconf(const SparkJob* msg); + static const ::flyteidl::core::K8sPod& driverpod(const SparkJob* msg); + static const ::flyteidl::core::K8sPod& executorpod(const SparkJob* msg); }; const ::google::protobuf::Struct& SparkJob::HasBitSetters::databricksconf(const SparkJob* msg) { return *msg->databricksconf_; } +const ::flyteidl::core::K8sPod& +SparkJob::HasBitSetters::driverpod(const SparkJob* msg) { + return *msg->driverPodValue_.driverpod_; +} +const ::flyteidl::core::K8sPod& +SparkJob::HasBitSetters::executorpod(const SparkJob* msg) { + return *msg->executorPodValue_.executorpod_; +} void SparkJob::clear_databricksconf() { if (GetArenaNoVirtual() == nullptr && databricksconf_ != nullptr) { delete databricksconf_; } databricksconf_ = nullptr; } +void SparkJob::set_allocated_driverpod(::flyteidl::core::K8sPod* driverpod) { + ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); + clear_driverPodValue(); + if (driverpod) { + ::google::protobuf::Arena* submessage_arena = nullptr; + if (message_arena != submessage_arena) { + driverpod = ::google::protobuf::internal::GetOwnedMessage( + message_arena, driverpod, submessage_arena); + } + set_has_driverpod(); + driverPodValue_.driverpod_ = driverpod; + } + // @@protoc_insertion_point(field_set_allocated:flyteidl.plugins.SparkJob.driverPod) +} +void SparkJob::clear_driverpod() { + if (has_driverpod()) { + delete driverPodValue_.driverpod_; + clear_has_driverPodValue(); + } +} +void SparkJob::set_allocated_executorpod(::flyteidl::core::K8sPod* executorpod) { + ::google::protobuf::Arena* message_arena = GetArenaNoVirtual(); + clear_executorPodValue(); + if (executorpod) { + ::google::protobuf::Arena* submessage_arena = nullptr; + if (message_arena != submessage_arena) { + executorpod = ::google::protobuf::internal::GetOwnedMessage( + message_arena, executorpod, submessage_arena); + } + set_has_executorpod(); + executorPodValue_.executorpod_ = executorpod; + } + // @@protoc_insertion_point(field_set_allocated:flyteidl.plugins.SparkJob.executorPod) +} +void SparkJob::clear_executorpod() { + if (has_executorpod()) { + delete executorPodValue_.executorpod_; + clear_has_executorPodValue(); + } +} #if !defined(_MSC_VER) || _MSC_VER >= 1900 const int SparkJob::kApplicationTypeFieldNumber; const int SparkJob::kMainApplicationFileFieldNumber; @@ -552,6 +622,10 @@ const int SparkJob::kExecutorPathFieldNumber; const int SparkJob::kDatabricksConfFieldNumber; const int SparkJob::kDatabricksTokenFieldNumber; const int SparkJob::kDatabricksInstanceFieldNumber; +const int SparkJob::kDriverPodFieldNumber; +const int SparkJob::kDriverPodTemplateNameFieldNumber; +const int SparkJob::kExecutorPodFieldNumber; +const int SparkJob::kExecutorPodTemplateNameFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 SparkJob::SparkJob() @@ -585,12 +659,40 @@ SparkJob::SparkJob(const SparkJob& from) if (from.databricksinstance().size() > 0) { databricksinstance_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.databricksinstance_); } + driverpodtemplatename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.driverpodtemplatename().size() > 0) { + driverpodtemplatename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.driverpodtemplatename_); + } + executorpodtemplatename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.executorpodtemplatename().size() > 0) { + executorpodtemplatename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.executorpodtemplatename_); + } if (from.has_databricksconf()) { databricksconf_ = new ::google::protobuf::Struct(*from.databricksconf_); } else { databricksconf_ = nullptr; } applicationtype_ = from.applicationtype_; + clear_has_driverPodValue(); + switch (from.driverPodValue_case()) { + case kDriverPod: { + mutable_driverpod()->::flyteidl::core::K8sPod::MergeFrom(from.driverpod()); + break; + } + case DRIVERPODVALUE_NOT_SET: { + break; + } + } + clear_has_executorPodValue(); + switch (from.executorPodValue_case()) { + case kExecutorPod: { + mutable_executorpod()->::flyteidl::core::K8sPod::MergeFrom(from.executorpod()); + break; + } + case EXECUTORPODVALUE_NOT_SET: { + break; + } + } // @@protoc_insertion_point(copy_constructor:flyteidl.plugins.SparkJob) } @@ -602,9 +704,13 @@ void SparkJob::SharedCtor() { executorpath_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databrickstoken_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databricksinstance_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + driverpodtemplatename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + executorpodtemplatename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); ::memset(&databricksconf_, 0, static_cast( reinterpret_cast(&applicationtype_) - reinterpret_cast(&databricksconf_)) + sizeof(applicationtype_)); + clear_has_driverPodValue(); + clear_has_executorPodValue(); } SparkJob::~SparkJob() { @@ -618,7 +724,15 @@ void SparkJob::SharedDtor() { executorpath_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databrickstoken_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databricksinstance_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + driverpodtemplatename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + executorpodtemplatename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete databricksconf_; + if (has_driverPodValue()) { + clear_driverPodValue(); + } + if (has_executorPodValue()) { + clear_executorPodValue(); + } } void SparkJob::SetCachedSize(int size) const { @@ -630,6 +744,35 @@ const SparkJob& SparkJob::default_instance() { } +void SparkJob::clear_driverPodValue() { +// @@protoc_insertion_point(one_of_clear_start:flyteidl.plugins.SparkJob) + switch (driverPodValue_case()) { + case kDriverPod: { + delete driverPodValue_.driverpod_; + break; + } + case DRIVERPODVALUE_NOT_SET: { + break; + } + } + _oneof_case_[0] = DRIVERPODVALUE_NOT_SET; +} + +void SparkJob::clear_executorPodValue() { +// @@protoc_insertion_point(one_of_clear_start:flyteidl.plugins.SparkJob) + switch (executorPodValue_case()) { + case kExecutorPod: { + delete executorPodValue_.executorpod_; + break; + } + case EXECUTORPODVALUE_NOT_SET: { + break; + } + } + _oneof_case_[1] = EXECUTORPODVALUE_NOT_SET; +} + + void SparkJob::Clear() { // @@protoc_insertion_point(message_clear_start:flyteidl.plugins.SparkJob) ::google::protobuf::uint32 cached_has_bits = 0; @@ -643,11 +786,15 @@ void SparkJob::Clear() { executorpath_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databrickstoken_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); databricksinstance_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + driverpodtemplatename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + executorpodtemplatename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && databricksconf_ != nullptr) { delete databricksconf_; } databricksconf_ = nullptr; applicationtype_ = 0; + clear_driverPodValue(); + clear_executorPodValue(); _internal_metadata_.Clear(); } @@ -803,6 +950,64 @@ const char* SparkJob::_InternalParse(const char* begin, const char* end, void* o ptr += size; break; } + // .flyteidl.core.K8sPod driverPod = 10; + case 10: { + if (static_cast<::google::protobuf::uint8>(tag) != 82) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + parser_till_end = ::flyteidl::core::K8sPod::_InternalParse; + object = msg->mutable_driverpod(); + if (size > end - ptr) goto len_delim_till_end; + ptr += size; + GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange( + {parser_till_end, object}, ptr - size, ptr)); + break; + } + // string driverPodTemplateName = 11; + case 11: { + if (static_cast<::google::protobuf::uint8>(tag) != 90) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + ctx->extra_parse_data().SetFieldName("flyteidl.plugins.SparkJob.driverPodTemplateName"); + object = msg->mutable_driverpodtemplatename(); + if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) { + parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8; + goto string_till_end; + } + GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx)); + ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx); + ptr += size; + break; + } + // .flyteidl.core.K8sPod executorPod = 12; + case 12: { + if (static_cast<::google::protobuf::uint8>(tag) != 98) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + parser_till_end = ::flyteidl::core::K8sPod::_InternalParse; + object = msg->mutable_executorpod(); + if (size > end - ptr) goto len_delim_till_end; + ptr += size; + GOOGLE_PROTOBUF_PARSER_ASSERT(ctx->ParseExactRange( + {parser_till_end, object}, ptr - size, ptr)); + break; + } + // string executorPodTemplateName = 13; + case 13: { + if (static_cast<::google::protobuf::uint8>(tag) != 106) goto handle_unusual; + ptr = ::google::protobuf::io::ReadSize(ptr, &size); + GOOGLE_PROTOBUF_PARSER_ASSERT(ptr); + ctx->extra_parse_data().SetFieldName("flyteidl.plugins.SparkJob.executorPodTemplateName"); + object = msg->mutable_executorpodtemplatename(); + if (size > end - ptr + ::google::protobuf::internal::ParseContext::kSlopBytes) { + parser_till_end = ::google::protobuf::internal::GreedyStringParserUTF8; + goto string_till_end; + } + GOOGLE_PROTOBUF_PARSER_ASSERT(::google::protobuf::internal::StringCheckUTF8(ptr, size, ctx)); + ::google::protobuf::internal::InlineGreedyStringParser(object, ptr, size, ctx); + ptr += size; + break; + } default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -989,6 +1194,58 @@ bool SparkJob::MergePartialFromCodedStream( break; } + // .flyteidl.core.K8sPod driverPod = 10; + case 10: { + if (static_cast< ::google::protobuf::uint8>(tag) == (82 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( + input, mutable_driverpod())); + } else { + goto handle_unusual; + } + break; + } + + // string driverPodTemplateName = 11; + case 11: { + if (static_cast< ::google::protobuf::uint8>(tag) == (90 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_driverpodtemplatename())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->driverpodtemplatename().data(), static_cast(this->driverpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "flyteidl.plugins.SparkJob.driverPodTemplateName")); + } else { + goto handle_unusual; + } + break; + } + + // .flyteidl.core.K8sPod executorPod = 12; + case 12: { + if (static_cast< ::google::protobuf::uint8>(tag) == (98 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadMessage( + input, mutable_executorpod())); + } else { + goto handle_unusual; + } + break; + } + + // string executorPodTemplateName = 13; + case 13: { + if (static_cast< ::google::protobuf::uint8>(tag) == (106 & 0xFF)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_executorpodtemplatename())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->executorpodtemplatename().data(), static_cast(this->executorpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "flyteidl.plugins.SparkJob.executorPodTemplateName")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1176,6 +1433,38 @@ void SparkJob::SerializeWithCachedSizes( 9, this->databricksinstance(), output); } + // .flyteidl.core.K8sPod driverPod = 10; + if (has_driverpod()) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 10, HasBitSetters::driverpod(this), output); + } + + // string driverPodTemplateName = 11; + if (this->driverpodtemplatename().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->driverpodtemplatename().data(), static_cast(this->driverpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.plugins.SparkJob.driverPodTemplateName"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 11, this->driverpodtemplatename(), output); + } + + // .flyteidl.core.K8sPod executorPod = 12; + if (has_executorpod()) { + ::google::protobuf::internal::WireFormatLite::WriteMessageMaybeToArray( + 12, HasBitSetters::executorpod(this), output); + } + + // string executorPodTemplateName = 13; + if (this->executorpodtemplatename().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->executorpodtemplatename().data(), static_cast(this->executorpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.plugins.SparkJob.executorPodTemplateName"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 13, this->executorpodtemplatename(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::google::protobuf::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1355,6 +1644,42 @@ ::google::protobuf::uint8* SparkJob::InternalSerializeWithCachedSizesToArray( 9, this->databricksinstance(), target); } + // .flyteidl.core.K8sPod driverPod = 10; + if (has_driverpod()) { + target = ::google::protobuf::internal::WireFormatLite:: + InternalWriteMessageToArray( + 10, HasBitSetters::driverpod(this), target); + } + + // string driverPodTemplateName = 11; + if (this->driverpodtemplatename().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->driverpodtemplatename().data(), static_cast(this->driverpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.plugins.SparkJob.driverPodTemplateName"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 11, this->driverpodtemplatename(), target); + } + + // .flyteidl.core.K8sPod executorPod = 12; + if (has_executorpod()) { + target = ::google::protobuf::internal::WireFormatLite:: + InternalWriteMessageToArray( + 12, HasBitSetters::executorpod(this), target); + } + + // string executorPodTemplateName = 13; + if (this->executorpodtemplatename().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->executorpodtemplatename().data(), static_cast(this->executorpodtemplatename().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "flyteidl.plugins.SparkJob.executorPodTemplateName"); + target = + ::google::protobuf::internal::WireFormatLite::WriteStringToArray( + 13, this->executorpodtemplatename(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::google::protobuf::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1439,6 +1764,20 @@ size_t SparkJob::ByteSizeLong() const { this->databricksinstance()); } + // string driverPodTemplateName = 11; + if (this->driverpodtemplatename().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->driverpodtemplatename()); + } + + // string executorPodTemplateName = 13; + if (this->executorpodtemplatename().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->executorpodtemplatename()); + } + // .google.protobuf.Struct databricksConf = 7; if (this->has_databricksconf()) { total_size += 1 + @@ -1452,6 +1791,30 @@ size_t SparkJob::ByteSizeLong() const { ::google::protobuf::internal::WireFormatLite::EnumSize(this->applicationtype()); } + switch (driverPodValue_case()) { + // .flyteidl.core.K8sPod driverPod = 10; + case kDriverPod: { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSize( + *driverPodValue_.driverpod_); + break; + } + case DRIVERPODVALUE_NOT_SET: { + break; + } + } + switch (executorPodValue_case()) { + // .flyteidl.core.K8sPod executorPod = 12; + case kExecutorPod: { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::MessageSize( + *executorPodValue_.executorpod_); + break; + } + case EXECUTORPODVALUE_NOT_SET: { + break; + } + } int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -1501,12 +1864,38 @@ void SparkJob::MergeFrom(const SparkJob& from) { databricksinstance_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.databricksinstance_); } + if (from.driverpodtemplatename().size() > 0) { + + driverpodtemplatename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.driverpodtemplatename_); + } + if (from.executorpodtemplatename().size() > 0) { + + executorpodtemplatename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.executorpodtemplatename_); + } if (from.has_databricksconf()) { mutable_databricksconf()->::google::protobuf::Struct::MergeFrom(from.databricksconf()); } if (from.applicationtype() != 0) { set_applicationtype(from.applicationtype()); } + switch (from.driverPodValue_case()) { + case kDriverPod: { + mutable_driverpod()->::flyteidl::core::K8sPod::MergeFrom(from.driverpod()); + break; + } + case DRIVERPODVALUE_NOT_SET: { + break; + } + } + switch (from.executorPodValue_case()) { + case kExecutorPod: { + mutable_executorpod()->::flyteidl::core::K8sPod::MergeFrom(from.executorpod()); + break; + } + case EXECUTORPODVALUE_NOT_SET: { + break; + } + } } void SparkJob::CopyFrom(const ::google::protobuf::Message& from) { @@ -1546,8 +1935,16 @@ void SparkJob::InternalSwap(SparkJob* other) { GetArenaNoVirtual()); databricksinstance_.Swap(&other->databricksinstance_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); + driverpodtemplatename_.Swap(&other->driverpodtemplatename_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); + executorpodtemplatename_.Swap(&other->executorpodtemplatename_, &::google::protobuf::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); swap(databricksconf_, other->databricksconf_); swap(applicationtype_, other->applicationtype_); + swap(driverPodValue_, other->driverPodValue_); + swap(executorPodValue_, other->executorPodValue_); + swap(_oneof_case_[0], other->_oneof_case_[0]); + swap(_oneof_case_[1], other->_oneof_case_[1]); } ::google::protobuf::Metadata SparkJob::GetMetadata() const { diff --git a/gen/pb-cpp/flyteidl/plugins/spark.pb.h b/gen/pb-cpp/flyteidl/plugins/spark.pb.h index 4b60371d8..c6ccc2b03 100644 --- a/gen/pb-cpp/flyteidl/plugins/spark.pb.h +++ b/gen/pb-cpp/flyteidl/plugins/spark.pb.h @@ -35,6 +35,7 @@ #include #include #include +#include "flyteidl/core/tasks.pb.h" #include // @@protoc_insertion_point(includes) #include @@ -320,6 +321,16 @@ class SparkJob final : } static const SparkJob& default_instance(); + enum DriverPodValueCase { + kDriverPod = 10, + DRIVERPODVALUE_NOT_SET = 0, + }; + + enum ExecutorPodValueCase { + kExecutorPod = 12, + EXECUTORPODVALUE_NOT_SET = 0, + }; + static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY static inline const SparkJob* internal_default_instance() { return reinterpret_cast( @@ -472,6 +483,34 @@ class SparkJob final : ::std::string* release_databricksinstance(); void set_allocated_databricksinstance(::std::string* databricksinstance); + // string driverPodTemplateName = 11; + void clear_driverpodtemplatename(); + static const int kDriverPodTemplateNameFieldNumber = 11; + const ::std::string& driverpodtemplatename() const; + void set_driverpodtemplatename(const ::std::string& value); + #if LANG_CXX11 + void set_driverpodtemplatename(::std::string&& value); + #endif + void set_driverpodtemplatename(const char* value); + void set_driverpodtemplatename(const char* value, size_t size); + ::std::string* mutable_driverpodtemplatename(); + ::std::string* release_driverpodtemplatename(); + void set_allocated_driverpodtemplatename(::std::string* driverpodtemplatename); + + // string executorPodTemplateName = 13; + void clear_executorpodtemplatename(); + static const int kExecutorPodTemplateNameFieldNumber = 13; + const ::std::string& executorpodtemplatename() const; + void set_executorpodtemplatename(const ::std::string& value); + #if LANG_CXX11 + void set_executorpodtemplatename(::std::string&& value); + #endif + void set_executorpodtemplatename(const char* value); + void set_executorpodtemplatename(const char* value, size_t size); + ::std::string* mutable_executorpodtemplatename(); + ::std::string* release_executorpodtemplatename(); + void set_allocated_executorpodtemplatename(::std::string* executorpodtemplatename); + // .google.protobuf.Struct databricksConf = 7; bool has_databricksconf() const; void clear_databricksconf(); @@ -487,9 +526,39 @@ class SparkJob final : ::flyteidl::plugins::SparkApplication_Type applicationtype() const; void set_applicationtype(::flyteidl::plugins::SparkApplication_Type value); + // .flyteidl.core.K8sPod driverPod = 10; + bool has_driverpod() const; + void clear_driverpod(); + static const int kDriverPodFieldNumber = 10; + const ::flyteidl::core::K8sPod& driverpod() const; + ::flyteidl::core::K8sPod* release_driverpod(); + ::flyteidl::core::K8sPod* mutable_driverpod(); + void set_allocated_driverpod(::flyteidl::core::K8sPod* driverpod); + + // .flyteidl.core.K8sPod executorPod = 12; + bool has_executorpod() const; + void clear_executorpod(); + static const int kExecutorPodFieldNumber = 12; + const ::flyteidl::core::K8sPod& executorpod() const; + ::flyteidl::core::K8sPod* release_executorpod(); + ::flyteidl::core::K8sPod* mutable_executorpod(); + void set_allocated_executorpod(::flyteidl::core::K8sPod* executorpod); + + void clear_driverPodValue(); + DriverPodValueCase driverPodValue_case() const; + void clear_executorPodValue(); + ExecutorPodValueCase executorPodValue_case() const; // @@protoc_insertion_point(class_scope:flyteidl.plugins.SparkJob) private: class HasBitSetters; + void set_has_driverpod(); + void set_has_executorpod(); + + inline bool has_driverPodValue() const; + inline void clear_has_driverPodValue(); + + inline bool has_executorPodValue() const; + inline void clear_has_executorPodValue(); ::google::protobuf::internal::InternalMetadataWithArena _internal_metadata_; ::google::protobuf::internal::MapField< @@ -509,9 +578,21 @@ class SparkJob final : ::google::protobuf::internal::ArenaStringPtr executorpath_; ::google::protobuf::internal::ArenaStringPtr databrickstoken_; ::google::protobuf::internal::ArenaStringPtr databricksinstance_; + ::google::protobuf::internal::ArenaStringPtr driverpodtemplatename_; + ::google::protobuf::internal::ArenaStringPtr executorpodtemplatename_; ::google::protobuf::Struct* databricksconf_; int applicationtype_; + union DriverPodValueUnion { + DriverPodValueUnion() {} + ::flyteidl::core::K8sPod* driverpod_; + } driverPodValue_; + union ExecutorPodValueUnion { + ExecutorPodValueUnion() {} + ::flyteidl::core::K8sPod* executorpod_; + } executorPodValue_; mutable ::google::protobuf::internal::CachedSize _cached_size_; + ::google::protobuf::uint32 _oneof_case_[2]; + friend struct ::TableStruct_flyteidl_2fplugins_2fspark_2eproto; }; // =================================================================== @@ -894,6 +975,200 @@ inline void SparkJob::set_allocated_databricksinstance(::std::string* databricks // @@protoc_insertion_point(field_set_allocated:flyteidl.plugins.SparkJob.databricksInstance) } +// .flyteidl.core.K8sPod driverPod = 10; +inline bool SparkJob::has_driverpod() const { + return driverPodValue_case() == kDriverPod; +} +inline void SparkJob::set_has_driverpod() { + _oneof_case_[0] = kDriverPod; +} +inline ::flyteidl::core::K8sPod* SparkJob::release_driverpod() { + // @@protoc_insertion_point(field_release:flyteidl.plugins.SparkJob.driverPod) + if (has_driverpod()) { + clear_has_driverPodValue(); + ::flyteidl::core::K8sPod* temp = driverPodValue_.driverpod_; + driverPodValue_.driverpod_ = nullptr; + return temp; + } else { + return nullptr; + } +} +inline const ::flyteidl::core::K8sPod& SparkJob::driverpod() const { + // @@protoc_insertion_point(field_get:flyteidl.plugins.SparkJob.driverPod) + return has_driverpod() + ? *driverPodValue_.driverpod_ + : *reinterpret_cast< ::flyteidl::core::K8sPod*>(&::flyteidl::core::_K8sPod_default_instance_); +} +inline ::flyteidl::core::K8sPod* SparkJob::mutable_driverpod() { + if (!has_driverpod()) { + clear_driverPodValue(); + set_has_driverpod(); + driverPodValue_.driverpod_ = CreateMaybeMessage< ::flyteidl::core::K8sPod >( + GetArenaNoVirtual()); + } + // @@protoc_insertion_point(field_mutable:flyteidl.plugins.SparkJob.driverPod) + return driverPodValue_.driverpod_; +} + +// string driverPodTemplateName = 11; +inline void SparkJob::clear_driverpodtemplatename() { + driverpodtemplatename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& SparkJob::driverpodtemplatename() const { + // @@protoc_insertion_point(field_get:flyteidl.plugins.SparkJob.driverPodTemplateName) + return driverpodtemplatename_.GetNoArena(); +} +inline void SparkJob::set_driverpodtemplatename(const ::std::string& value) { + + driverpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:flyteidl.plugins.SparkJob.driverPodTemplateName) +} +#if LANG_CXX11 +inline void SparkJob::set_driverpodtemplatename(::std::string&& value) { + + driverpodtemplatename_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:flyteidl.plugins.SparkJob.driverPodTemplateName) +} +#endif +inline void SparkJob::set_driverpodtemplatename(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + driverpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:flyteidl.plugins.SparkJob.driverPodTemplateName) +} +inline void SparkJob::set_driverpodtemplatename(const char* value, size_t size) { + + driverpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:flyteidl.plugins.SparkJob.driverPodTemplateName) +} +inline ::std::string* SparkJob::mutable_driverpodtemplatename() { + + // @@protoc_insertion_point(field_mutable:flyteidl.plugins.SparkJob.driverPodTemplateName) + return driverpodtemplatename_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* SparkJob::release_driverpodtemplatename() { + // @@protoc_insertion_point(field_release:flyteidl.plugins.SparkJob.driverPodTemplateName) + + return driverpodtemplatename_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void SparkJob::set_allocated_driverpodtemplatename(::std::string* driverpodtemplatename) { + if (driverpodtemplatename != nullptr) { + + } else { + + } + driverpodtemplatename_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), driverpodtemplatename); + // @@protoc_insertion_point(field_set_allocated:flyteidl.plugins.SparkJob.driverPodTemplateName) +} + +// .flyteidl.core.K8sPod executorPod = 12; +inline bool SparkJob::has_executorpod() const { + return executorPodValue_case() == kExecutorPod; +} +inline void SparkJob::set_has_executorpod() { + _oneof_case_[1] = kExecutorPod; +} +inline ::flyteidl::core::K8sPod* SparkJob::release_executorpod() { + // @@protoc_insertion_point(field_release:flyteidl.plugins.SparkJob.executorPod) + if (has_executorpod()) { + clear_has_executorPodValue(); + ::flyteidl::core::K8sPod* temp = executorPodValue_.executorpod_; + executorPodValue_.executorpod_ = nullptr; + return temp; + } else { + return nullptr; + } +} +inline const ::flyteidl::core::K8sPod& SparkJob::executorpod() const { + // @@protoc_insertion_point(field_get:flyteidl.plugins.SparkJob.executorPod) + return has_executorpod() + ? *executorPodValue_.executorpod_ + : *reinterpret_cast< ::flyteidl::core::K8sPod*>(&::flyteidl::core::_K8sPod_default_instance_); +} +inline ::flyteidl::core::K8sPod* SparkJob::mutable_executorpod() { + if (!has_executorpod()) { + clear_executorPodValue(); + set_has_executorpod(); + executorPodValue_.executorpod_ = CreateMaybeMessage< ::flyteidl::core::K8sPod >( + GetArenaNoVirtual()); + } + // @@protoc_insertion_point(field_mutable:flyteidl.plugins.SparkJob.executorPod) + return executorPodValue_.executorpod_; +} + +// string executorPodTemplateName = 13; +inline void SparkJob::clear_executorpodtemplatename() { + executorpodtemplatename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& SparkJob::executorpodtemplatename() const { + // @@protoc_insertion_point(field_get:flyteidl.plugins.SparkJob.executorPodTemplateName) + return executorpodtemplatename_.GetNoArena(); +} +inline void SparkJob::set_executorpodtemplatename(const ::std::string& value) { + + executorpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:flyteidl.plugins.SparkJob.executorPodTemplateName) +} +#if LANG_CXX11 +inline void SparkJob::set_executorpodtemplatename(::std::string&& value) { + + executorpodtemplatename_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:flyteidl.plugins.SparkJob.executorPodTemplateName) +} +#endif +inline void SparkJob::set_executorpodtemplatename(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + executorpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:flyteidl.plugins.SparkJob.executorPodTemplateName) +} +inline void SparkJob::set_executorpodtemplatename(const char* value, size_t size) { + + executorpodtemplatename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:flyteidl.plugins.SparkJob.executorPodTemplateName) +} +inline ::std::string* SparkJob::mutable_executorpodtemplatename() { + + // @@protoc_insertion_point(field_mutable:flyteidl.plugins.SparkJob.executorPodTemplateName) + return executorpodtemplatename_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* SparkJob::release_executorpodtemplatename() { + // @@protoc_insertion_point(field_release:flyteidl.plugins.SparkJob.executorPodTemplateName) + + return executorpodtemplatename_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void SparkJob::set_allocated_executorpodtemplatename(::std::string* executorpodtemplatename) { + if (executorpodtemplatename != nullptr) { + + } else { + + } + executorpodtemplatename_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), executorpodtemplatename); + // @@protoc_insertion_point(field_set_allocated:flyteidl.plugins.SparkJob.executorPodTemplateName) +} + +inline bool SparkJob::has_driverPodValue() const { + return driverPodValue_case() != DRIVERPODVALUE_NOT_SET; +} +inline void SparkJob::clear_has_driverPodValue() { + _oneof_case_[0] = DRIVERPODVALUE_NOT_SET; +} +inline bool SparkJob::has_executorPodValue() const { + return executorPodValue_case() != EXECUTORPODVALUE_NOT_SET; +} +inline void SparkJob::clear_has_executorPodValue() { + _oneof_case_[1] = EXECUTORPODVALUE_NOT_SET; +} +inline SparkJob::DriverPodValueCase SparkJob::driverPodValue_case() const { + return SparkJob::DriverPodValueCase(_oneof_case_[0]); +} +inline SparkJob::ExecutorPodValueCase SparkJob::executorPodValue_case() const { + return SparkJob::ExecutorPodValueCase(_oneof_case_[1]); +} #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ diff --git a/gen/pb-go/flyteidl/plugins/spark.pb.go b/gen/pb-go/flyteidl/plugins/spark.pb.go index 768bd46ba..33468989a 100644 --- a/gen/pb-go/flyteidl/plugins/spark.pb.go +++ b/gen/pb-go/flyteidl/plugins/spark.pb.go @@ -5,6 +5,7 @@ package plugins import ( fmt "fmt" + core "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/core" proto "github.com/golang/protobuf/proto" _struct "github.com/golang/protobuf/ptypes/struct" math "math" @@ -99,10 +100,32 @@ type SparkJob struct { DatabricksToken string `protobuf:"bytes,8,opt,name=databricksToken,proto3" json:"databricksToken,omitempty"` // Domain name of your deployment. Use the form .cloud.databricks.com. // This instance name can be set in either flytepropeller or flytekit. - DatabricksInstance string `protobuf:"bytes,9,opt,name=databricksInstance,proto3" json:"databricksInstance,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + DatabricksInstance string `protobuf:"bytes,9,opt,name=databricksInstance,proto3" json:"databricksInstance,omitempty"` + // The pod spec and metadata to be used as the base configuration when creating the driver Pod for this task. + // +optional + // + // Types that are valid to be assigned to DriverPodValue: + // *SparkJob_DriverPod + DriverPodValue isSparkJob_DriverPodValue `protobuf_oneof:"driverPodValue"` + // Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the + // driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied + // identically as, the default PodTemplate configured in FlytePropeller. + // +optional + DriverPodTemplateName string `protobuf:"bytes,11,opt,name=driverPodTemplateName,proto3" json:"driverPodTemplateName,omitempty"` + // The pod spec and metadata to be used as the base configuration when creating the executor Pods for this task. + // +optional + // + // Types that are valid to be assigned to ExecutorPodValue: + // *SparkJob_ExecutorPod + ExecutorPodValue isSparkJob_ExecutorPodValue `protobuf_oneof:"executorPodValue"` + // Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the + // executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied + // identically as, the default PodTemplate configured in FlytePropeller. + // +optional + ExecutorPodTemplateName string `protobuf:"bytes,13,opt,name=executorPodTemplateName,proto3" json:"executorPodTemplateName,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *SparkJob) Reset() { *m = SparkJob{} } @@ -193,6 +216,76 @@ func (m *SparkJob) GetDatabricksInstance() string { return "" } +type isSparkJob_DriverPodValue interface { + isSparkJob_DriverPodValue() +} + +type SparkJob_DriverPod struct { + DriverPod *core.K8SPod `protobuf:"bytes,10,opt,name=driverPod,proto3,oneof"` +} + +func (*SparkJob_DriverPod) isSparkJob_DriverPodValue() {} + +func (m *SparkJob) GetDriverPodValue() isSparkJob_DriverPodValue { + if m != nil { + return m.DriverPodValue + } + return nil +} + +func (m *SparkJob) GetDriverPod() *core.K8SPod { + if x, ok := m.GetDriverPodValue().(*SparkJob_DriverPod); ok { + return x.DriverPod + } + return nil +} + +func (m *SparkJob) GetDriverPodTemplateName() string { + if m != nil { + return m.DriverPodTemplateName + } + return "" +} + +type isSparkJob_ExecutorPodValue interface { + isSparkJob_ExecutorPodValue() +} + +type SparkJob_ExecutorPod struct { + ExecutorPod *core.K8SPod `protobuf:"bytes,12,opt,name=executorPod,proto3,oneof"` +} + +func (*SparkJob_ExecutorPod) isSparkJob_ExecutorPodValue() {} + +func (m *SparkJob) GetExecutorPodValue() isSparkJob_ExecutorPodValue { + if m != nil { + return m.ExecutorPodValue + } + return nil +} + +func (m *SparkJob) GetExecutorPod() *core.K8SPod { + if x, ok := m.GetExecutorPodValue().(*SparkJob_ExecutorPod); ok { + return x.ExecutorPod + } + return nil +} + +func (m *SparkJob) GetExecutorPodTemplateName() string { + if m != nil { + return m.ExecutorPodTemplateName + } + return "" +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*SparkJob) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*SparkJob_DriverPod)(nil), + (*SparkJob_ExecutorPod)(nil), + } +} + func init() { proto.RegisterEnum("flyteidl.plugins.SparkApplication_Type", SparkApplication_Type_name, SparkApplication_Type_value) proto.RegisterType((*SparkApplication)(nil), "flyteidl.plugins.SparkApplication") @@ -204,33 +297,40 @@ func init() { func init() { proto.RegisterFile("flyteidl/plugins/spark.proto", fileDescriptor_ca8a069b9820144a) } var fileDescriptor_ca8a069b9820144a = []byte{ - // 442 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8b, 0xd3, 0x40, - 0x14, 0x35, 0xfd, 0xb2, 0xb9, 0x2b, 0x6d, 0xb8, 0x0a, 0x86, 0xd2, 0x87, 0xd2, 0x17, 0xa3, 0xe0, - 0x44, 0xea, 0x83, 0x1f, 0x28, 0x92, 0x2d, 0xea, 0x5a, 0x44, 0xd7, 0xb4, 0x08, 0xfa, 0x36, 0x49, - 0xa7, 0x69, 0xe8, 0xec, 0xcc, 0x90, 0x4c, 0xc4, 0xfc, 0x79, 0x91, 0x4c, 0xec, 0x66, 0x37, 0xac, - 0x82, 0x6f, 0xc9, 0xb9, 0xe7, 0x9c, 0x7b, 0x39, 0x87, 0x81, 0xe9, 0x8e, 0x97, 0x9a, 0xa5, 0x5b, - 0xee, 0x2b, 0x5e, 0x24, 0xa9, 0xc8, 0xfd, 0x5c, 0xd1, 0xec, 0x40, 0x54, 0x26, 0xb5, 0x44, 0xe7, - 0x38, 0x25, 0x7f, 0xa6, 0x93, 0x69, 0x22, 0x65, 0xc2, 0x99, 0x6f, 0xe6, 0x51, 0xb1, 0xf3, 0x73, - 0x9d, 0x15, 0xb1, 0xae, 0xf9, 0xf3, 0x53, 0x70, 0xd6, 0x95, 0x3c, 0x50, 0x8a, 0xa7, 0x31, 0xd5, - 0xa9, 0x14, 0x73, 0x02, 0xbd, 0x4d, 0xa9, 0x18, 0x02, 0x0c, 0xce, 0xbf, 0x6d, 0xce, 0x3e, 0x7f, - 0x72, 0x6e, 0xe1, 0x10, 0x7a, 0xab, 0xe0, 0x6b, 0xe0, 0x58, 0x68, 0x43, 0x7f, 0xbd, 0x0c, 0x3e, - 0x06, 0x4e, 0x07, 0xfb, 0x60, 0x85, 0x4e, 0x77, 0xfe, 0xab, 0x07, 0x43, 0x63, 0xb2, 0x92, 0x11, - 0x7e, 0x81, 0x31, 0x6d, 0xbc, 0x2a, 0x1f, 0xd7, 0x9a, 0x59, 0xde, 0x68, 0xf1, 0x80, 0xb4, 0x4f, - 0x23, 0xed, 0xcd, 0xa4, 0xa2, 0x87, 0x6d, 0x3d, 0x3e, 0x81, 0xbb, 0x17, 0x34, 0x15, 0x57, 0x88, - 0xef, 0x52, 0xce, 0xdc, 0xce, 0xcc, 0xf2, 0xec, 0xf0, 0xa6, 0x11, 0x4e, 0xc1, 0xae, 0xe0, 0x25, - 0xa7, 0x79, 0xee, 0x76, 0x0d, 0xaf, 0x01, 0xf0, 0x3d, 0xd8, 0x26, 0xb2, 0xa5, 0x14, 0x3b, 0xb7, - 0x37, 0xeb, 0x7a, 0x27, 0x8b, 0x87, 0x7f, 0x39, 0x6e, 0x25, 0xa3, 0xfa, 0xa3, 0xe2, 0xbe, 0x15, - 0x3a, 0x2b, 0xc3, 0x46, 0x8b, 0x2b, 0x80, 0x3d, 0xdd, 0x4a, 0xa9, 0x8c, 0x53, 0xdf, 0x38, 0x3d, - 0xfa, 0x87, 0xd3, 0xd9, 0x25, 0xb9, 0xb6, 0xba, 0xa2, 0xc6, 0x39, 0xdc, 0x61, 0x3f, 0x59, 0x5c, - 0x68, 0x99, 0x9d, 0x53, 0xbd, 0x77, 0x07, 0xe6, 0xea, 0x6b, 0x18, 0xbe, 0x81, 0xd1, 0x96, 0x6a, - 0x1a, 0x65, 0x69, 0x7c, 0xc8, 0xcd, 0xce, 0xdb, 0x33, 0xcb, 0x3b, 0x59, 0xdc, 0x27, 0x75, 0xc7, - 0xe4, 0xd8, 0x31, 0x59, 0x9b, 0x8e, 0xc3, 0x16, 0x1d, 0x3d, 0x18, 0x37, 0xc8, 0x46, 0x1e, 0x98, - 0x70, 0x87, 0x66, 0x4f, 0x1b, 0x46, 0x02, 0xd8, 0x40, 0x1f, 0x44, 0xae, 0xa9, 0x88, 0x99, 0x6b, - 0x1b, 0xf2, 0x0d, 0x93, 0xc9, 0x2b, 0x18, 0x5d, 0xcf, 0x09, 0x1d, 0xe8, 0x1e, 0x58, 0x69, 0xca, - 0xb7, 0xc3, 0xea, 0x13, 0xef, 0x41, 0xff, 0x07, 0xe5, 0xc5, 0xb1, 0xb9, 0xfa, 0xe7, 0x65, 0xe7, - 0xb9, 0x35, 0x79, 0x0d, 0xe3, 0x56, 0x36, 0xff, 0x23, 0x3f, 0x7d, 0xf1, 0xfd, 0x59, 0x92, 0xea, - 0x7d, 0x11, 0x91, 0x58, 0x5e, 0xf8, 0x26, 0x7f, 0x99, 0x25, 0xfe, 0xe5, 0x43, 0x49, 0x98, 0xf0, - 0x55, 0xf4, 0x38, 0x91, 0x7e, 0xfb, 0xed, 0x44, 0x03, 0x13, 0xd9, 0xd3, 0xdf, 0x01, 0x00, 0x00, - 0xff, 0xff, 0x02, 0x51, 0x7a, 0xdc, 0x56, 0x03, 0x00, 0x00, + // 547 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0x51, 0x8f, 0xd2, 0x40, + 0x10, 0xbe, 0x1e, 0xc7, 0xdd, 0x75, 0x38, 0xb9, 0x66, 0xf4, 0x72, 0x95, 0xf0, 0x40, 0x78, 0x11, + 0x4d, 0xdc, 0x1a, 0xd4, 0xc8, 0x19, 0x8d, 0x01, 0xa2, 0x22, 0x9a, 0x13, 0x0b, 0xb9, 0x44, 0xdf, + 0xb6, 0xed, 0x52, 0x1a, 0x4a, 0xb7, 0x69, 0xb7, 0x17, 0xf9, 0x03, 0xfe, 0x6e, 0xd3, 0xad, 0xb4, + 0xd0, 0x70, 0x97, 0xf8, 0xb6, 0xf3, 0xcd, 0x37, 0xdf, 0x37, 0x99, 0x99, 0x2c, 0x34, 0xe7, 0xfe, + 0x5a, 0x30, 0xcf, 0xf1, 0x8d, 0xd0, 0x4f, 0x5c, 0x2f, 0x88, 0x8d, 0x38, 0xa4, 0xd1, 0x92, 0x84, + 0x11, 0x17, 0x1c, 0xb5, 0x4d, 0x96, 0xfc, 0xcb, 0x36, 0x1e, 0xe7, 0x7c, 0x9b, 0x47, 0xcc, 0x10, + 0x34, 0x5e, 0xc6, 0x19, 0xb9, 0xd1, 0x74, 0x39, 0x77, 0x7d, 0x66, 0xc8, 0xc8, 0x4a, 0xe6, 0x46, + 0x2c, 0xa2, 0xc4, 0x16, 0x59, 0xb6, 0x3d, 0x00, 0x6d, 0x9a, 0x2a, 0xf7, 0xc3, 0xd0, 0xf7, 0x6c, + 0x2a, 0x3c, 0x1e, 0xb4, 0x09, 0x1c, 0xcd, 0xd6, 0x21, 0x43, 0x80, 0xe3, 0xc9, 0xcf, 0xd9, 0xe8, + 0xfb, 0xb5, 0x76, 0x80, 0xa7, 0x70, 0x34, 0xee, 0xdf, 0xf4, 0x35, 0x05, 0x55, 0xa8, 0x4e, 0x87, + 0xfd, 0x6f, 0x7d, 0xed, 0x10, 0xab, 0xa0, 0x98, 0x5a, 0xa5, 0xfd, 0xe7, 0x04, 0x4e, 0xa5, 0xc8, + 0x98, 0x5b, 0xf8, 0x03, 0xce, 0x69, 0xa1, 0x95, 0xea, 0xe8, 0x4a, 0x4b, 0xe9, 0xd4, 0xbb, 0x4f, + 0x48, 0xb9, 0x6b, 0x52, 0x76, 0x26, 0x29, 0xdd, 0x2c, 0xd7, 0xe3, 0x0b, 0x78, 0xb8, 0xa2, 0x5e, + 0xb0, 0x45, 0xfc, 0xe4, 0xf9, 0x4c, 0x3f, 0x6c, 0x29, 0x1d, 0xd5, 0xdc, 0x97, 0xc2, 0x26, 0xa8, + 0x29, 0x3c, 0xf4, 0x69, 0x1c, 0xeb, 0x15, 0xc9, 0x2b, 0x00, 0xfc, 0x0c, 0xaa, 0x9c, 0xe6, 0x90, + 0x07, 0x73, 0xfd, 0xa8, 0x55, 0xe9, 0xd4, 0xba, 0x4f, 0xef, 0x68, 0x6e, 0xcc, 0xad, 0xec, 0x91, + 0x72, 0x3f, 0x06, 0x22, 0x5a, 0x9b, 0x45, 0x2d, 0x8e, 0x01, 0x16, 0xd4, 0xe1, 0x3c, 0x94, 0x4a, + 0x55, 0xa9, 0xf4, 0xec, 0x1e, 0xa5, 0x51, 0x4e, 0xce, 0xa4, 0xb6, 0xaa, 0xb1, 0x0d, 0x67, 0xec, + 0x37, 0xb3, 0x13, 0xc1, 0xa3, 0x09, 0x15, 0x0b, 0xfd, 0x58, 0x76, 0xbd, 0x83, 0xe1, 0x07, 0xa8, + 0x3b, 0x54, 0x50, 0x2b, 0xf2, 0xec, 0x65, 0x2c, 0x3d, 0x4f, 0x5a, 0x4a, 0xa7, 0xd6, 0xbd, 0x24, + 0xd9, 0x8e, 0xc9, 0x66, 0xc7, 0x64, 0x2a, 0x77, 0x6c, 0x96, 0xe8, 0xd8, 0x81, 0xf3, 0x02, 0x99, + 0xf1, 0x25, 0x0b, 0xf4, 0x53, 0xe9, 0x53, 0x86, 0x91, 0x00, 0x16, 0xd0, 0x97, 0x20, 0x16, 0x34, + 0xb0, 0x99, 0xae, 0x4a, 0xf2, 0x9e, 0x0c, 0xbe, 0x06, 0xd5, 0x89, 0xbc, 0x5b, 0x16, 0x4d, 0xb8, + 0xa3, 0x83, 0xec, 0xea, 0xa2, 0x98, 0x44, 0x7a, 0x94, 0xe4, 0x6b, 0x2f, 0x9e, 0x70, 0x67, 0x74, + 0x60, 0x16, 0x4c, 0x7c, 0x05, 0x17, 0x79, 0x30, 0x63, 0xab, 0xd0, 0xa7, 0x82, 0x5d, 0xd3, 0x15, + 0xd3, 0x6b, 0xd2, 0x69, 0x7f, 0x12, 0xaf, 0xa0, 0x96, 0xcf, 0x85, 0x3b, 0xfa, 0xd9, 0x7d, 0x76, + 0x8a, 0xb9, 0xcd, 0xc5, 0x1e, 0x5c, 0x6e, 0x85, 0x3b, 0x96, 0x0f, 0xa4, 0xe5, 0x5d, 0xe9, 0xc6, + 0x3b, 0xa8, 0xef, 0x5e, 0x02, 0x6a, 0x50, 0x59, 0xb2, 0xb5, 0x3c, 0x6f, 0xd5, 0x4c, 0x9f, 0xf8, + 0x08, 0xaa, 0xb7, 0xd4, 0x4f, 0x36, 0xb7, 0x99, 0x05, 0x6f, 0x0f, 0x7b, 0x4a, 0xe3, 0x3d, 0x9c, + 0x97, 0xb6, 0xff, 0x3f, 0xe5, 0x03, 0x0d, 0xea, 0xf9, 0x28, 0x6e, 0x52, 0x74, 0x80, 0xa0, 0x6d, + 0x75, 0x9a, 0x61, 0x57, 0xbf, 0xde, 0xb8, 0x9e, 0x58, 0x24, 0x16, 0xb1, 0xf9, 0xca, 0x90, 0xe3, + 0xe0, 0x91, 0x6b, 0xe4, 0x7f, 0x83, 0xcb, 0x02, 0x23, 0xb4, 0x9e, 0xbb, 0xdc, 0x28, 0x7f, 0x2f, + 0xd6, 0xb1, 0x3c, 0x9d, 0x97, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x4f, 0x9f, 0xa7, 0x79, + 0x04, 0x00, 0x00, } diff --git a/gen/pb-go/flyteidl/plugins/spark.pb.validate.go b/gen/pb-go/flyteidl/plugins/spark.pb.validate.go index 0577090e2..1e7d7c5f7 100644 --- a/gen/pb-go/flyteidl/plugins/spark.pb.validate.go +++ b/gen/pb-go/flyteidl/plugins/spark.pb.validate.go @@ -134,6 +134,42 @@ func (m *SparkJob) Validate() error { // no validation rules for DatabricksInstance + // no validation rules for DriverPodTemplateName + + // no validation rules for ExecutorPodTemplateName + + switch m.DriverPodValue.(type) { + + case *SparkJob_DriverPod: + + if v, ok := interface{}(m.GetDriverPod()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SparkJobValidationError{ + field: "DriverPod", + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + switch m.ExecutorPodValue.(type) { + + case *SparkJob_ExecutorPod: + + if v, ok := interface{}(m.GetExecutorPod()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SparkJobValidationError{ + field: "ExecutorPod", + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + return nil } diff --git a/gen/pb-java/flyteidl/plugins/Spark.java b/gen/pb-java/flyteidl/plugins/Spark.java index 57bfefee8..b8d84fe05 100644 --- a/gen/pb-java/flyteidl/plugins/Spark.java +++ b/gen/pb-java/flyteidl/plugins/Spark.java @@ -727,6 +727,84 @@ java.lang.String getHadoopConfOrThrow( */ com.google.protobuf.ByteString getDatabricksInstanceBytes(); + + /** + * .flyteidl.core.K8sPod driverPod = 10; + */ + boolean hasDriverPod(); + /** + * .flyteidl.core.K8sPod driverPod = 10; + */ + flyteidl.core.Tasks.K8sPod getDriverPod(); + /** + * .flyteidl.core.K8sPod driverPod = 10; + */ + flyteidl.core.Tasks.K8sPodOrBuilder getDriverPodOrBuilder(); + + /** + * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     *
+ * + * string driverPodTemplateName = 11; + */ + java.lang.String getDriverPodTemplateName(); + /** + * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     *
+ * + * string driverPodTemplateName = 11; + */ + com.google.protobuf.ByteString + getDriverPodTemplateNameBytes(); + + /** + * .flyteidl.core.K8sPod executorPod = 12; + */ + boolean hasExecutorPod(); + /** + * .flyteidl.core.K8sPod executorPod = 12; + */ + flyteidl.core.Tasks.K8sPod getExecutorPod(); + /** + * .flyteidl.core.K8sPod executorPod = 12; + */ + flyteidl.core.Tasks.K8sPodOrBuilder getExecutorPodOrBuilder(); + + /** + * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     *
+ * + * string executorPodTemplateName = 13; + */ + java.lang.String getExecutorPodTemplateName(); + /** + * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     *
+ * + * string executorPodTemplateName = 13; + */ + com.google.protobuf.ByteString + getExecutorPodTemplateNameBytes(); + + public flyteidl.plugins.Spark.SparkJob.DriverPodValueCase getDriverPodValueCase(); + + public flyteidl.plugins.Spark.SparkJob.ExecutorPodValueCase getExecutorPodValueCase(); } /** * 
@@ -751,6 +829,8 @@ private SparkJob() {
       executorPath_ = "";
       databricksToken_ = "";
       databricksInstance_ = "";
+      driverPodTemplateName_ = "";
+      executorPodTemplateName_ = "";
     }
 
     @java.lang.Override
@@ -852,6 +932,46 @@ private SparkJob(
               databricksInstance_ = s;
               break;
             }
+            case 82: {
+              flyteidl.core.Tasks.K8sPod.Builder subBuilder = null;
+              if (driverPodValueCase_ == 10) {
+                subBuilder = ((flyteidl.core.Tasks.K8sPod) driverPodValue_).toBuilder();
+              }
+              driverPodValue_ =
+                  input.readMessage(flyteidl.core.Tasks.K8sPod.parser(), extensionRegistry);
+              if (subBuilder != null) {
+                subBuilder.mergeFrom((flyteidl.core.Tasks.K8sPod) driverPodValue_);
+                driverPodValue_ = subBuilder.buildPartial();
+              }
+              driverPodValueCase_ = 10;
+              break;
+            }
+            case 90: {
+              java.lang.String s = input.readStringRequireUtf8();
+
+              driverPodTemplateName_ = s;
+              break;
+            }
+            case 98: {
+              flyteidl.core.Tasks.K8sPod.Builder subBuilder = null;
+              if (executorPodValueCase_ == 12) {
+                subBuilder = ((flyteidl.core.Tasks.K8sPod) executorPodValue_).toBuilder();
+              }
+              executorPodValue_ =
+                  input.readMessage(flyteidl.core.Tasks.K8sPod.parser(), extensionRegistry);
+              if (subBuilder != null) {
+                subBuilder.mergeFrom((flyteidl.core.Tasks.K8sPod) executorPodValue_);
+                executorPodValue_ = subBuilder.buildPartial();
+              }
+              executorPodValueCase_ = 12;
+              break;
+            }
+            case 106: {
+              java.lang.String s = input.readStringRequireUtf8();
+
+              executorPodTemplateName_ = s;
+              break;
+            }
             default: {
               if (!parseUnknownField(
                   input, unknownFields, extensionRegistry, tag)) {
@@ -899,6 +1019,78 @@ protected com.google.protobuf.MapField internalGetMapField(
     }
 
     private int bitField0_;
+    private int driverPodValueCase_ = 0;
+    private java.lang.Object driverPodValue_;
+    public enum DriverPodValueCase
+        implements com.google.protobuf.Internal.EnumLite {
+      DRIVERPOD(10),
+      DRIVERPODVALUE_NOT_SET(0);
+      private final int value;
+      private DriverPodValueCase(int value) {
+        this.value = value;
+      }
+      /**
+       * @deprecated Use {@link #forNumber(int)} instead.
+       */
+      @java.lang.Deprecated
+      public static DriverPodValueCase valueOf(int value) {
+        return forNumber(value);
+      }
+
+      public static DriverPodValueCase forNumber(int value) {
+        switch (value) {
+          case 10: return DRIVERPOD;
+          case 0: return DRIVERPODVALUE_NOT_SET;
+          default: return null;
+        }
+      }
+      public int getNumber() {
+        return this.value;
+      }
+    };
+
+    public DriverPodValueCase
+    getDriverPodValueCase() {
+      return DriverPodValueCase.forNumber(
+          driverPodValueCase_);
+    }
+
+    private int executorPodValueCase_ = 0;
+    private java.lang.Object executorPodValue_;
+    public enum ExecutorPodValueCase
+        implements com.google.protobuf.Internal.EnumLite {
+      EXECUTORPOD(12),
+      EXECUTORPODVALUE_NOT_SET(0);
+      private final int value;
+      private ExecutorPodValueCase(int value) {
+        this.value = value;
+      }
+      /**
+       * @deprecated Use {@link #forNumber(int)} instead.
+       */
+      @java.lang.Deprecated
+      public static ExecutorPodValueCase valueOf(int value) {
+        return forNumber(value);
+      }
+
+      public static ExecutorPodValueCase forNumber(int value) {
+        switch (value) {
+          case 12: return EXECUTORPOD;
+          case 0: return EXECUTORPODVALUE_NOT_SET;
+          default: return null;
+        }
+      }
+      public int getNumber() {
+        return this.value;
+      }
+    };
+
+    public ExecutorPodValueCase
+    getExecutorPodValueCase() {
+      return ExecutorPodValueCase.forNumber(
+          executorPodValueCase_);
+    }
+
     public static final int APPLICATIONTYPE_FIELD_NUMBER = 1;
     private int applicationType_;
     /**
@@ -1302,6 +1494,154 @@ public java.lang.String getDatabricksInstance() {
       }
     }
 
+    public static final int DRIVERPOD_FIELD_NUMBER = 10;
+    /**
+     * .flyteidl.core.K8sPod driverPod = 10;
+     */
+    public boolean hasDriverPod() {
+      return driverPodValueCase_ == 10;
+    }
+    /**
+     * .flyteidl.core.K8sPod driverPod = 10;
+     */
+    public flyteidl.core.Tasks.K8sPod getDriverPod() {
+      if (driverPodValueCase_ == 10) {
+         return (flyteidl.core.Tasks.K8sPod) driverPodValue_;
+      }
+      return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+    }
+    /**
+     * .flyteidl.core.K8sPod driverPod = 10;
+     */
+    public flyteidl.core.Tasks.K8sPodOrBuilder getDriverPodOrBuilder() {
+      if (driverPodValueCase_ == 10) {
+         return (flyteidl.core.Tasks.K8sPod) driverPodValue_;
+      }
+      return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+    }
+
+    public static final int DRIVERPODTEMPLATENAME_FIELD_NUMBER = 11;
+    private volatile java.lang.Object driverPodTemplateName_;
+    /**
+     * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     * 

+     *
+     * string driverPodTemplateName = 11;
+     */
+    public java.lang.String getDriverPodTemplateName() {
+      java.lang.Object ref = driverPodTemplateName_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        driverPodTemplateName_ = s;
+        return s;
+      }
+    }
+    /**
+     * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     * 

+     *
+     * string driverPodTemplateName = 11;
+     */
+    public com.google.protobuf.ByteString
+        getDriverPodTemplateNameBytes() {
+      java.lang.Object ref = driverPodTemplateName_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        driverPodTemplateName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
+    public static final int EXECUTORPOD_FIELD_NUMBER = 12;
+    /**
+     * .flyteidl.core.K8sPod executorPod = 12;
+     */
+    public boolean hasExecutorPod() {
+      return executorPodValueCase_ == 12;
+    }
+    /**
+     * .flyteidl.core.K8sPod executorPod = 12;
+     */
+    public flyteidl.core.Tasks.K8sPod getExecutorPod() {
+      if (executorPodValueCase_ == 12) {
+         return (flyteidl.core.Tasks.K8sPod) executorPodValue_;
+      }
+      return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+    }
+    /**
+     * .flyteidl.core.K8sPod executorPod = 12;
+     */
+    public flyteidl.core.Tasks.K8sPodOrBuilder getExecutorPodOrBuilder() {
+      if (executorPodValueCase_ == 12) {
+         return (flyteidl.core.Tasks.K8sPod) executorPodValue_;
+      }
+      return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+    }
+
+    public static final int EXECUTORPODTEMPLATENAME_FIELD_NUMBER = 13;
+    private volatile java.lang.Object executorPodTemplateName_;
+    /**
+     * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     * 

+     *
+     * string executorPodTemplateName = 13;
+     */
+    public java.lang.String getExecutorPodTemplateName() {
+      java.lang.Object ref = executorPodTemplateName_;
+      if (ref instanceof java.lang.String) {
+        return (java.lang.String) ref;
+      } else {
+        com.google.protobuf.ByteString bs = 
+            (com.google.protobuf.ByteString) ref;
+        java.lang.String s = bs.toStringUtf8();
+        executorPodTemplateName_ = s;
+        return s;
+      }
+    }
+    /**
+     * 
+     * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+     * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+     * identically as, the default PodTemplate configured in FlytePropeller.
+     * +optional
+     * 

+     *
+     * string executorPodTemplateName = 13;
+     */
+    public com.google.protobuf.ByteString
+        getExecutorPodTemplateNameBytes() {
+      java.lang.Object ref = executorPodTemplateName_;
+      if (ref instanceof java.lang.String) {
+        com.google.protobuf.ByteString b = 
+            com.google.protobuf.ByteString.copyFromUtf8(
+                (java.lang.String) ref);
+        executorPodTemplateName_ = b;
+        return b;
+      } else {
+        return (com.google.protobuf.ByteString) ref;
+      }
+    }
+
     private byte memoizedIsInitialized = -1;
     @java.lang.Override
     public final boolean isInitialized() {
@@ -1349,6 +1689,18 @@ public void writeTo(com.google.protobuf.CodedOutputStream output)
       if (!getDatabricksInstanceBytes().isEmpty()) {
         com.google.protobuf.GeneratedMessageV3.writeString(output, 9, databricksInstance_);
       }
+      if (driverPodValueCase_ == 10) {
+        output.writeMessage(10, (flyteidl.core.Tasks.K8sPod) driverPodValue_);
+      }
+      if (!getDriverPodTemplateNameBytes().isEmpty()) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 11, driverPodTemplateName_);
+      }
+      if (executorPodValueCase_ == 12) {
+        output.writeMessage(12, (flyteidl.core.Tasks.K8sPod) executorPodValue_);
+      }
+      if (!getExecutorPodTemplateNameBytes().isEmpty()) {
+        com.google.protobuf.GeneratedMessageV3.writeString(output, 13, executorPodTemplateName_);
+      }
       unknownFields.writeTo(output);
     }
 
@@ -1401,6 +1753,20 @@ public int getSerializedSize() {
       if (!getDatabricksInstanceBytes().isEmpty()) {
         size += com.google.protobuf.GeneratedMessageV3.computeStringSize(9, databricksInstance_);
       }
+      if (driverPodValueCase_ == 10) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(10, (flyteidl.core.Tasks.K8sPod) driverPodValue_);
+      }
+      if (!getDriverPodTemplateNameBytes().isEmpty()) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(11, driverPodTemplateName_);
+      }
+      if (executorPodValueCase_ == 12) {
+        size += com.google.protobuf.CodedOutputStream
+          .computeMessageSize(12, (flyteidl.core.Tasks.K8sPod) executorPodValue_);
+      }
+      if (!getExecutorPodTemplateNameBytes().isEmpty()) {
+        size += com.google.protobuf.GeneratedMessageV3.computeStringSize(13, executorPodTemplateName_);
+      }
       size += unknownFields.getSerializedSize();
       memoizedSize = size;
       return size;
@@ -1436,6 +1802,28 @@ public boolean equals(final java.lang.Object obj) {
           .equals(other.getDatabricksToken())) return false;
       if (!getDatabricksInstance()
           .equals(other.getDatabricksInstance())) return false;
+      if (!getDriverPodTemplateName()
+          .equals(other.getDriverPodTemplateName())) return false;
+      if (!getExecutorPodTemplateName()
+          .equals(other.getExecutorPodTemplateName())) return false;
+      if (!getDriverPodValueCase().equals(other.getDriverPodValueCase())) return false;
+      switch (driverPodValueCase_) {
+        case 10:
+          if (!getDriverPod()
+              .equals(other.getDriverPod())) return false;
+          break;
+        case 0:
+        default:
+      }
+      if (!getExecutorPodValueCase().equals(other.getExecutorPodValueCase())) return false;
+      switch (executorPodValueCase_) {
+        case 12:
+          if (!getExecutorPod()
+              .equals(other.getExecutorPod())) return false;
+          break;
+        case 0:
+        default:
+      }
       if (!unknownFields.equals(other.unknownFields)) return false;
       return true;
     }
@@ -1471,6 +1859,26 @@ public int hashCode() {
       hash = (53 * hash) + getDatabricksToken().hashCode();
       hash = (37 * hash) + DATABRICKSINSTANCE_FIELD_NUMBER;
       hash = (53 * hash) + getDatabricksInstance().hashCode();
+      hash = (37 * hash) + DRIVERPODTEMPLATENAME_FIELD_NUMBER;
+      hash = (53 * hash) + getDriverPodTemplateName().hashCode();
+      hash = (37 * hash) + EXECUTORPODTEMPLATENAME_FIELD_NUMBER;
+      hash = (53 * hash) + getExecutorPodTemplateName().hashCode();
+      switch (driverPodValueCase_) {
+        case 10:
+          hash = (37 * hash) + DRIVERPOD_FIELD_NUMBER;
+          hash = (53 * hash) + getDriverPod().hashCode();
+          break;
+        case 0:
+        default:
+      }
+      switch (executorPodValueCase_) {
+        case 12:
+          hash = (37 * hash) + EXECUTORPOD_FIELD_NUMBER;
+          hash = (53 * hash) + getExecutorPod().hashCode();
+          break;
+        case 0:
+        default:
+      }
       hash = (29 * hash) + unknownFields.hashCode();
       memoizedHashCode = hash;
       return hash;
@@ -1654,6 +2062,14 @@ public Builder clear() {
 
         databricksInstance_ = "";
 
+        driverPodTemplateName_ = "";
+
+        executorPodTemplateName_ = "";
+
+        driverPodValueCase_ = 0;
+        driverPodValue_ = null;
+        executorPodValueCase_ = 0;
+        executorPodValue_ = null;
         return this;
       }
 
@@ -1697,7 +2113,25 @@ public flyteidl.plugins.Spark.SparkJob buildPartial() {
         }
         result.databricksToken_ = databricksToken_;
         result.databricksInstance_ = databricksInstance_;
+        if (driverPodValueCase_ == 10) {
+          if (driverPodBuilder_ == null) {
+            result.driverPodValue_ = driverPodValue_;
+          } else {
+            result.driverPodValue_ = driverPodBuilder_.build();
+          }
+        }
+        result.driverPodTemplateName_ = driverPodTemplateName_;
+        if (executorPodValueCase_ == 12) {
+          if (executorPodBuilder_ == null) {
+            result.executorPodValue_ = executorPodValue_;
+          } else {
+            result.executorPodValue_ = executorPodBuilder_.build();
+          }
+        }
+        result.executorPodTemplateName_ = executorPodTemplateName_;
         result.bitField0_ = to_bitField0_;
+        result.driverPodValueCase_ = driverPodValueCase_;
+        result.executorPodValueCase_ = executorPodValueCase_;
         onBuilt();
         return result;
       }
@@ -1776,6 +2210,32 @@ public Builder mergeFrom(flyteidl.plugins.Spark.SparkJob other) {
           databricksInstance_ = other.databricksInstance_;
           onChanged();
         }
+        if (!other.getDriverPodTemplateName().isEmpty()) {
+          driverPodTemplateName_ = other.driverPodTemplateName_;
+          onChanged();
+        }
+        if (!other.getExecutorPodTemplateName().isEmpty()) {
+          executorPodTemplateName_ = other.executorPodTemplateName_;
+          onChanged();
+        }
+        switch (other.getDriverPodValueCase()) {
+          case DRIVERPOD: {
+            mergeDriverPod(other.getDriverPod());
+            break;
+          }
+          case DRIVERPODVALUE_NOT_SET: {
+            break;
+          }
+        }
+        switch (other.getExecutorPodValueCase()) {
+          case EXECUTORPOD: {
+            mergeExecutorPod(other.getExecutorPod());
+            break;
+          }
+          case EXECUTORPODVALUE_NOT_SET: {
+            break;
+          }
+        }
         this.mergeUnknownFields(other.unknownFields);
         onChanged();
         return this;
@@ -1804,6 +2264,36 @@ public Builder mergeFrom(
         }
         return this;
       }
+      private int driverPodValueCase_ = 0;
+      private java.lang.Object driverPodValue_;
+      public DriverPodValueCase
+          getDriverPodValueCase() {
+        return DriverPodValueCase.forNumber(
+            driverPodValueCase_);
+      }
+
+      public Builder clearDriverPodValue() {
+        driverPodValueCase_ = 0;
+        driverPodValue_ = null;
+        onChanged();
+        return this;
+      }
+
+      private int executorPodValueCase_ = 0;
+      private java.lang.Object executorPodValue_;
+      public ExecutorPodValueCase
+          getExecutorPodValueCase() {
+        return ExecutorPodValueCase.forNumber(
+            executorPodValueCase_);
+      }
+
+      public Builder clearExecutorPodValue() {
+        executorPodValueCase_ = 0;
+        executorPodValue_ = null;
+        onChanged();
+        return this;
+      }
+
       private int bitField0_;
 
       private int applicationType_ = 0;
@@ -2673,6 +3163,486 @@ public Builder setDatabricksInstanceBytes(
         onChanged();
         return this;
       }
+
+      private com.google.protobuf.SingleFieldBuilderV3<
+          flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder> driverPodBuilder_;
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public boolean hasDriverPod() {
+        return driverPodValueCase_ == 10;
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public flyteidl.core.Tasks.K8sPod getDriverPod() {
+        if (driverPodBuilder_ == null) {
+          if (driverPodValueCase_ == 10) {
+            return (flyteidl.core.Tasks.K8sPod) driverPodValue_;
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        } else {
+          if (driverPodValueCase_ == 10) {
+            return driverPodBuilder_.getMessage();
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        }
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public Builder setDriverPod(flyteidl.core.Tasks.K8sPod value) {
+        if (driverPodBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          driverPodValue_ = value;
+          onChanged();
+        } else {
+          driverPodBuilder_.setMessage(value);
+        }
+        driverPodValueCase_ = 10;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public Builder setDriverPod(
+          flyteidl.core.Tasks.K8sPod.Builder builderForValue) {
+        if (driverPodBuilder_ == null) {
+          driverPodValue_ = builderForValue.build();
+          onChanged();
+        } else {
+          driverPodBuilder_.setMessage(builderForValue.build());
+        }
+        driverPodValueCase_ = 10;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public Builder mergeDriverPod(flyteidl.core.Tasks.K8sPod value) {
+        if (driverPodBuilder_ == null) {
+          if (driverPodValueCase_ == 10 &&
+              driverPodValue_ != flyteidl.core.Tasks.K8sPod.getDefaultInstance()) {
+            driverPodValue_ = flyteidl.core.Tasks.K8sPod.newBuilder((flyteidl.core.Tasks.K8sPod) driverPodValue_)
+                .mergeFrom(value).buildPartial();
+          } else {
+            driverPodValue_ = value;
+          }
+          onChanged();
+        } else {
+          if (driverPodValueCase_ == 10) {
+            driverPodBuilder_.mergeFrom(value);
+          }
+          driverPodBuilder_.setMessage(value);
+        }
+        driverPodValueCase_ = 10;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public Builder clearDriverPod() {
+        if (driverPodBuilder_ == null) {
+          if (driverPodValueCase_ == 10) {
+            driverPodValueCase_ = 0;
+            driverPodValue_ = null;
+            onChanged();
+          }
+        } else {
+          if (driverPodValueCase_ == 10) {
+            driverPodValueCase_ = 0;
+            driverPodValue_ = null;
+          }
+          driverPodBuilder_.clear();
+        }
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public flyteidl.core.Tasks.K8sPod.Builder getDriverPodBuilder() {
+        return getDriverPodFieldBuilder().getBuilder();
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      public flyteidl.core.Tasks.K8sPodOrBuilder getDriverPodOrBuilder() {
+        if ((driverPodValueCase_ == 10) && (driverPodBuilder_ != null)) {
+          return driverPodBuilder_.getMessageOrBuilder();
+        } else {
+          if (driverPodValueCase_ == 10) {
+            return (flyteidl.core.Tasks.K8sPod) driverPodValue_;
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        }
+      }
+      /**
+       * .flyteidl.core.K8sPod driverPod = 10;
+       */
+      private com.google.protobuf.SingleFieldBuilderV3<
+          flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder> 
+          getDriverPodFieldBuilder() {
+        if (driverPodBuilder_ == null) {
+          if (!(driverPodValueCase_ == 10)) {
+            driverPodValue_ = flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+          }
+          driverPodBuilder_ = new com.google.protobuf.SingleFieldBuilderV3<
+              flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder>(
+                  (flyteidl.core.Tasks.K8sPod) driverPodValue_,
+                  getParentForChildren(),
+                  isClean());
+          driverPodValue_ = null;
+        }
+        driverPodValueCase_ = 10;
+        onChanged();;
+        return driverPodBuilder_;
+      }
+
+      private java.lang.Object driverPodTemplateName_ = "";
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string driverPodTemplateName = 11;
+       */
+      public java.lang.String getDriverPodTemplateName() {
+        java.lang.Object ref = driverPodTemplateName_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs =
+              (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          driverPodTemplateName_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string driverPodTemplateName = 11;
+       */
+      public com.google.protobuf.ByteString
+          getDriverPodTemplateNameBytes() {
+        java.lang.Object ref = driverPodTemplateName_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          driverPodTemplateName_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string driverPodTemplateName = 11;
+       */
+      public Builder setDriverPodTemplateName(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  
+        driverPodTemplateName_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string driverPodTemplateName = 11;
+       */
+      public Builder clearDriverPodTemplateName() {
+        
+        driverPodTemplateName_ = getDefaultInstance().getDriverPodTemplateName();
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string driverPodTemplateName = 11;
+       */
+      public Builder setDriverPodTemplateNameBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+        
+        driverPodTemplateName_ = value;
+        onChanged();
+        return this;
+      }
+
+      private com.google.protobuf.SingleFieldBuilderV3<
+          flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder> executorPodBuilder_;
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public boolean hasExecutorPod() {
+        return executorPodValueCase_ == 12;
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public flyteidl.core.Tasks.K8sPod getExecutorPod() {
+        if (executorPodBuilder_ == null) {
+          if (executorPodValueCase_ == 12) {
+            return (flyteidl.core.Tasks.K8sPod) executorPodValue_;
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        } else {
+          if (executorPodValueCase_ == 12) {
+            return executorPodBuilder_.getMessage();
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        }
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public Builder setExecutorPod(flyteidl.core.Tasks.K8sPod value) {
+        if (executorPodBuilder_ == null) {
+          if (value == null) {
+            throw new NullPointerException();
+          }
+          executorPodValue_ = value;
+          onChanged();
+        } else {
+          executorPodBuilder_.setMessage(value);
+        }
+        executorPodValueCase_ = 12;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public Builder setExecutorPod(
+          flyteidl.core.Tasks.K8sPod.Builder builderForValue) {
+        if (executorPodBuilder_ == null) {
+          executorPodValue_ = builderForValue.build();
+          onChanged();
+        } else {
+          executorPodBuilder_.setMessage(builderForValue.build());
+        }
+        executorPodValueCase_ = 12;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public Builder mergeExecutorPod(flyteidl.core.Tasks.K8sPod value) {
+        if (executorPodBuilder_ == null) {
+          if (executorPodValueCase_ == 12 &&
+              executorPodValue_ != flyteidl.core.Tasks.K8sPod.getDefaultInstance()) {
+            executorPodValue_ = flyteidl.core.Tasks.K8sPod.newBuilder((flyteidl.core.Tasks.K8sPod) executorPodValue_)
+                .mergeFrom(value).buildPartial();
+          } else {
+            executorPodValue_ = value;
+          }
+          onChanged();
+        } else {
+          if (executorPodValueCase_ == 12) {
+            executorPodBuilder_.mergeFrom(value);
+          }
+          executorPodBuilder_.setMessage(value);
+        }
+        executorPodValueCase_ = 12;
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public Builder clearExecutorPod() {
+        if (executorPodBuilder_ == null) {
+          if (executorPodValueCase_ == 12) {
+            executorPodValueCase_ = 0;
+            executorPodValue_ = null;
+            onChanged();
+          }
+        } else {
+          if (executorPodValueCase_ == 12) {
+            executorPodValueCase_ = 0;
+            executorPodValue_ = null;
+          }
+          executorPodBuilder_.clear();
+        }
+        return this;
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public flyteidl.core.Tasks.K8sPod.Builder getExecutorPodBuilder() {
+        return getExecutorPodFieldBuilder().getBuilder();
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      public flyteidl.core.Tasks.K8sPodOrBuilder getExecutorPodOrBuilder() {
+        if ((executorPodValueCase_ == 12) && (executorPodBuilder_ != null)) {
+          return executorPodBuilder_.getMessageOrBuilder();
+        } else {
+          if (executorPodValueCase_ == 12) {
+            return (flyteidl.core.Tasks.K8sPod) executorPodValue_;
+          }
+          return flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+        }
+      }
+      /**
+       * .flyteidl.core.K8sPod executorPod = 12;
+       */
+      private com.google.protobuf.SingleFieldBuilderV3<
+          flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder> 
+          getExecutorPodFieldBuilder() {
+        if (executorPodBuilder_ == null) {
+          if (!(executorPodValueCase_ == 12)) {
+            executorPodValue_ = flyteidl.core.Tasks.K8sPod.getDefaultInstance();
+          }
+          executorPodBuilder_ = new com.google.protobuf.SingleFieldBuilderV3<
+              flyteidl.core.Tasks.K8sPod, flyteidl.core.Tasks.K8sPod.Builder, flyteidl.core.Tasks.K8sPodOrBuilder>(
+                  (flyteidl.core.Tasks.K8sPod) executorPodValue_,
+                  getParentForChildren(),
+                  isClean());
+          executorPodValue_ = null;
+        }
+        executorPodValueCase_ = 12;
+        onChanged();;
+        return executorPodBuilder_;
+      }
+
+      private java.lang.Object executorPodTemplateName_ = "";
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string executorPodTemplateName = 13;
+       */
+      public java.lang.String getExecutorPodTemplateName() {
+        java.lang.Object ref = executorPodTemplateName_;
+        if (!(ref instanceof java.lang.String)) {
+          com.google.protobuf.ByteString bs =
+              (com.google.protobuf.ByteString) ref;
+          java.lang.String s = bs.toStringUtf8();
+          executorPodTemplateName_ = s;
+          return s;
+        } else {
+          return (java.lang.String) ref;
+        }
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string executorPodTemplateName = 13;
+       */
+      public com.google.protobuf.ByteString
+          getExecutorPodTemplateNameBytes() {
+        java.lang.Object ref = executorPodTemplateName_;
+        if (ref instanceof String) {
+          com.google.protobuf.ByteString b = 
+              com.google.protobuf.ByteString.copyFromUtf8(
+                  (java.lang.String) ref);
+          executorPodTemplateName_ = b;
+          return b;
+        } else {
+          return (com.google.protobuf.ByteString) ref;
+        }
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string executorPodTemplateName = 13;
+       */
+      public Builder setExecutorPodTemplateName(
+          java.lang.String value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  
+        executorPodTemplateName_ = value;
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string executorPodTemplateName = 13;
+       */
+      public Builder clearExecutorPodTemplateName() {
+        
+        executorPodTemplateName_ = getDefaultInstance().getExecutorPodTemplateName();
+        onChanged();
+        return this;
+      }
+      /**
+       * 
+       * Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+       * executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+       * identically as, the default PodTemplate configured in FlytePropeller.
+       * +optional
+       * 

+       *
+       * string executorPodTemplateName = 13;
+       */
+      public Builder setExecutorPodTemplateNameBytes(
+          com.google.protobuf.ByteString value) {
+        if (value == null) {
+    throw new NullPointerException();
+  }
+  checkByteStringIsUtf8(value);
+        
+        executorPodTemplateName_ = value;
+        onChanged();
+        return this;
+      }
       @java.lang.Override
       public final Builder setUnknownFields(
           final com.google.protobuf.UnknownFieldSet unknownFields) {
@@ -2756,23 +3726,28 @@ public flyteidl.plugins.Spark.SparkJob getDefaultInstanceForType() {
   static {
     java.lang.String[] descriptorData = {
       "\n\034flyteidl/plugins/spark.proto\022\020flyteidl" +
-      ".plugins\032\034google/protobuf/struct.proto\"B" +
-      "\n\020SparkApplication\".\n\004Type\022\n\n\006PYTHON\020\000\022\010" +
-      "\n\004JAVA\020\001\022\t\n\005SCALA\020\002\022\005\n\001R\020\003\"\333\003\n\010SparkJob\022" +
-      "@\n\017applicationType\030\001 \001(\0162\'.flyteidl.plug" +
-      "ins.SparkApplication.Type\022\033\n\023mainApplica" +
-      "tionFile\030\002 \001(\t\022\021\n\tmainClass\030\003 \001(\t\022<\n\tspa" +
-      "rkConf\030\004 \003(\0132).flyteidl.plugins.SparkJob" +
-      ".SparkConfEntry\022>\n\nhadoopConf\030\005 \003(\0132*.fl" +
-      "yteidl.plugins.SparkJob.HadoopConfEntry\022" +
-      "\024\n\014executorPath\030\006 \001(\t\022/\n\016databricksConf\030" +
-      "\007 \001(\0132\027.google.protobuf.Struct\022\027\n\017databr" +
-      "icksToken\030\010 \001(\t\022\032\n\022databricksInstance\030\t " +
-      "\001(\t\0320\n\016SparkConfEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005va" +
-      "lue\030\002 \001(\t:\0028\001\0321\n\017HadoopConfEntry\022\013\n\003key\030" +
-      "\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001B9Z7github.com/f" +
-      "lyteorg/flyteidl/gen/pb-go/flyteidl/plug" +
-      "insb\006proto3"
+      ".plugins\032\031flyteidl/core/tasks.proto\032\034goo" +
+      "gle/protobuf/struct.proto\"B\n\020SparkApplic" +
+      "ation\".\n\004Type\022\n\n\006PYTHON\020\000\022\010\n\004JAVA\020\001\022\t\n\005S" +
+      "CALA\020\002\022\005\n\001R\020\003\"\233\005\n\010SparkJob\022@\n\017applicatio" +
+      "nType\030\001 \001(\0162\'.flyteidl.plugins.SparkAppl" +
+      "ication.Type\022\033\n\023mainApplicationFile\030\002 \001(" +
+      "\t\022\021\n\tmainClass\030\003 \001(\t\022<\n\tsparkConf\030\004 \003(\0132" +
+      ").flyteidl.plugins.SparkJob.SparkConfEnt" +
+      "ry\022>\n\nhadoopConf\030\005 \003(\0132*.flyteidl.plugin" +
+      "s.SparkJob.HadoopConfEntry\022\024\n\014executorPa" +
+      "th\030\006 \001(\t\022/\n\016databricksConf\030\007 \001(\0132\027.googl" +
+      "e.protobuf.Struct\022\027\n\017databricksToken\030\010 \001" +
+      "(\t\022\032\n\022databricksInstance\030\t \001(\t\022*\n\tdriver" +
+      "Pod\030\n \001(\0132\025.flyteidl.core.K8sPodH\000\022\035\n\025dr" +
+      "iverPodTemplateName\030\013 \001(\t\022,\n\013executorPod" +
+      "\030\014 \001(\0132\025.flyteidl.core.K8sPodH\001\022\037\n\027execu" +
+      "torPodTemplateName\030\r \001(\t\0320\n\016SparkConfEnt" +
+      "ry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\0321\n\017Ha" +
+      "doopConfEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(" +
+      "\t:\0028\001B\020\n\016driverPodValueB\022\n\020executorPodVa" +
+      "lueB9Z7github.com/flyteorg/flyteidl/gen/" +
+      "pb-go/flyteidl/pluginsb\006proto3"
     };
     com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
         new com.google.protobuf.Descriptors.FileDescriptor.    InternalDescriptorAssigner() {
@@ -2785,6 +3760,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
     com.google.protobuf.Descriptors.FileDescriptor
       .internalBuildGeneratedFileFrom(descriptorData,
         new com.google.protobuf.Descriptors.FileDescriptor[] {
+          flyteidl.core.Tasks.getDescriptor(),
           com.google.protobuf.StructProto.getDescriptor(),
         }, assigner);
     internal_static_flyteidl_plugins_SparkApplication_descriptor =
@@ -2798,7 +3774,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
     internal_static_flyteidl_plugins_SparkJob_fieldAccessorTable = new
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_flyteidl_plugins_SparkJob_descriptor,
-        new java.lang.String[] { "ApplicationType", "MainApplicationFile", "MainClass", "SparkConf", "HadoopConf", "ExecutorPath", "DatabricksConf", "DatabricksToken", "DatabricksInstance", });
+        new java.lang.String[] { "ApplicationType", "MainApplicationFile", "MainClass", "SparkConf", "HadoopConf", "ExecutorPath", "DatabricksConf", "DatabricksToken", "DatabricksInstance", "DriverPod", "DriverPodTemplateName", "ExecutorPod", "ExecutorPodTemplateName", "DriverPodValue", "ExecutorPodValue", });
     internal_static_flyteidl_plugins_SparkJob_SparkConfEntry_descriptor =
       internal_static_flyteidl_plugins_SparkJob_descriptor.getNestedTypes().get(0);
     internal_static_flyteidl_plugins_SparkJob_SparkConfEntry_fieldAccessorTable = new
@@ -2811,6 +3787,7 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors(
       com.google.protobuf.GeneratedMessageV3.FieldAccessorTable(
         internal_static_flyteidl_plugins_SparkJob_HadoopConfEntry_descriptor,
         new java.lang.String[] { "Key", "Value", });
+    flyteidl.core.Tasks.getDescriptor();
     com.google.protobuf.StructProto.getDescriptor();
   }
 
diff --git a/gen/pb_python/flyteidl/plugins/spark_pb2.py b/gen/pb_python/flyteidl/plugins/spark_pb2.py
index dc319cf62..9095c7d1d 100644
--- a/gen/pb_python/flyteidl/plugins/spark_pb2.py
+++ b/gen/pb_python/flyteidl/plugins/spark_pb2.py
@@ -11,10 +11,11 @@
 _sym_db = _symbol_database.Default()
 
 
+from flyteidl.core import tasks_pb2 as flyteidl_dot_core_dot_tasks__pb2
 from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2
 
 
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/plugins/spark.proto\x12\x10\x66lyteidl.plugins\x1a\x1cgoogle/protobuf/struct.proto\"B\n\x10SparkApplication\".\n\x04Type\x12\n\n\x06PYTHON\x10\x00\x12\x08\n\x04JAVA\x10\x01\x12\t\n\x05SCALA\x10\x02\x12\x05\n\x01R\x10\x03\"\xfe\x04\n\x08SparkJob\x12Q\n\x0f\x61pplicationType\x18\x01 \x01(\x0e\x32\'.flyteidl.plugins.SparkApplication.TypeR\x0f\x61pplicationType\x12\x30\n\x13mainApplicationFile\x18\x02 \x01(\tR\x13mainApplicationFile\x12\x1c\n\tmainClass\x18\x03 \x01(\tR\tmainClass\x12G\n\tsparkConf\x18\x04 \x03(\x0b\x32).flyteidl.plugins.SparkJob.SparkConfEntryR\tsparkConf\x12J\n\nhadoopConf\x18\x05 \x03(\x0b\x32*.flyteidl.plugins.SparkJob.HadoopConfEntryR\nhadoopConf\x12\"\n\x0c\x65xecutorPath\x18\x06 \x01(\tR\x0c\x65xecutorPath\x12?\n\x0e\x64\x61tabricksConf\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x0e\x64\x61tabricksConf\x12(\n\x0f\x64\x61tabricksToken\x18\x08 \x01(\tR\x0f\x64\x61tabricksToken\x12.\n\x12\x64\x61tabricksInstance\x18\t \x01(\tR\x12\x64\x61tabricksInstance\x1a<\n\x0eSparkConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a=\n\x0fHadoopConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\xbc\x01\n\x14\x63om.flyteidl.pluginsB\nSparkProtoP\x01Z7github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins\xa2\x02\x03\x46PX\xaa\x02\x10\x46lyteidl.Plugins\xca\x02\x10\x46lyteidl\\Plugins\xe2\x02\x1c\x46lyteidl\\Plugins\\GPBMetadata\xea\x02\x11\x46lyteidl::Pluginsb\x06proto3')
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66lyteidl/plugins/spark.proto\x12\x10\x66lyteidl.plugins\x1a\x19\x66lyteidl/core/tasks.proto\x1a\x1cgoogle/protobuf/struct.proto\"B\n\x10SparkApplication\".\n\x04Type\x12\n\n\x06PYTHON\x10\x00\x12\x08\n\x04JAVA\x10\x01\x12\t\n\x05SCALA\x10\x02\x12\x05\n\x01R\x10\x03\"\x86\x07\n\x08SparkJob\x12Q\n\x0f\x61pplicationType\x18\x01 \x01(\x0e\x32\'.flyteidl.plugins.SparkApplication.TypeR\x0f\x61pplicationType\x12\x30\n\x13mainApplicationFile\x18\x02 \x01(\tR\x13mainApplicationFile\x12\x1c\n\tmainClass\x18\x03 \x01(\tR\tmainClass\x12G\n\tsparkConf\x18\x04 \x03(\x0b\x32).flyteidl.plugins.SparkJob.SparkConfEntryR\tsparkConf\x12J\n\nhadoopConf\x18\x05 \x03(\x0b\x32*.flyteidl.plugins.SparkJob.HadoopConfEntryR\nhadoopConf\x12\"\n\x0c\x65xecutorPath\x18\x06 \x01(\tR\x0c\x65xecutorPath\x12?\n\x0e\x64\x61tabricksConf\x18\x07 \x01(\x0b\x32\x17.google.protobuf.StructR\x0e\x64\x61tabricksConf\x12(\n\x0f\x64\x61tabricksToken\x18\x08 \x01(\tR\x0f\x64\x61tabricksToken\x12.\n\x12\x64\x61tabricksInstance\x18\t \x01(\tR\x12\x64\x61tabricksInstance\x12\x35\n\tdriverPod\x18\n \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x00R\tdriverPod\x12\x34\n\x15\x64riverPodTemplateName\x18\x0b \x01(\tR\x15\x64riverPodTemplateName\x12\x39\n\x0b\x65xecutorPod\x18\x0c \x01(\x0b\x32\x15.flyteidl.core.K8sPodH\x01R\x0b\x65xecutorPod\x12\x38\n\x17\x65xecutorPodTemplateName\x18\r \x01(\tR\x17\x65xecutorPodTemplateName\x1a<\n\x0eSparkConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a=\n\x0fHadoopConfEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\x10\n\x0e\x64riverPodValueB\x12\n\x10\x65xecutorPodValueB\xbc\x01\n\x14\x63om.flyteidl.pluginsB\nSparkProtoP\x01Z7github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins\xa2\x02\x03\x46PX\xaa\x02\x10\x46lyteidl.Plugins\xca\x02\x10\x46lyteidl\\Plugins\xe2\x02\x1c\x46lyteidl\\Plugins\\GPBMetadata\xea\x02\x11\x46lyteidl::Pluginsb\x06proto3')
 
 _globals = globals()
 _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
@@ -27,14 +28,14 @@
   _SPARKJOB_SPARKCONFENTRY._serialized_options = b'8\001'
   _SPARKJOB_HADOOPCONFENTRY._options = None
   _SPARKJOB_HADOOPCONFENTRY._serialized_options = b'8\001'
-  _globals['_SPARKAPPLICATION']._serialized_start=80
-  _globals['_SPARKAPPLICATION']._serialized_end=146
-  _globals['_SPARKAPPLICATION_TYPE']._serialized_start=100
-  _globals['_SPARKAPPLICATION_TYPE']._serialized_end=146
-  _globals['_SPARKJOB']._serialized_start=149
-  _globals['_SPARKJOB']._serialized_end=787
-  _globals['_SPARKJOB_SPARKCONFENTRY']._serialized_start=664
-  _globals['_SPARKJOB_SPARKCONFENTRY']._serialized_end=724
-  _globals['_SPARKJOB_HADOOPCONFENTRY']._serialized_start=726
-  _globals['_SPARKJOB_HADOOPCONFENTRY']._serialized_end=787
+  _globals['_SPARKAPPLICATION']._serialized_start=107
+  _globals['_SPARKAPPLICATION']._serialized_end=173
+  _globals['_SPARKAPPLICATION_TYPE']._serialized_start=127
+  _globals['_SPARKAPPLICATION_TYPE']._serialized_end=173
+  _globals['_SPARKJOB']._serialized_start=176
+  _globals['_SPARKJOB']._serialized_end=1078
+  _globals['_SPARKJOB_SPARKCONFENTRY']._serialized_start=917
+  _globals['_SPARKJOB_SPARKCONFENTRY']._serialized_end=977
+  _globals['_SPARKJOB_HADOOPCONFENTRY']._serialized_start=979
+  _globals['_SPARKJOB_HADOOPCONFENTRY']._serialized_end=1040
 # @@protoc_insertion_point(module_scope)
diff --git a/gen/pb_python/flyteidl/plugins/spark_pb2.pyi b/gen/pb_python/flyteidl/plugins/spark_pb2.pyi
index e6b9e4eb6..95029a261 100644
--- a/gen/pb_python/flyteidl/plugins/spark_pb2.pyi
+++ b/gen/pb_python/flyteidl/plugins/spark_pb2.pyi
@@ -1,3 +1,4 @@
+from flyteidl.core import tasks_pb2 as _tasks_pb2
 from google.protobuf import struct_pb2 as _struct_pb2
 from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
@@ -22,7 +23,7 @@ class SparkApplication(_message.Message):
     def __init__(self) -> None: ...
 
 class SparkJob(_message.Message):
-    __slots__ = ["applicationType", "mainApplicationFile", "mainClass", "sparkConf", "hadoopConf", "executorPath", "databricksConf", "databricksToken", "databricksInstance"]
+    __slots__ = ["applicationType", "mainApplicationFile", "mainClass", "sparkConf", "hadoopConf", "executorPath", "databricksConf", "databricksToken", "databricksInstance", "driverPod", "driverPodTemplateName", "executorPod", "executorPodTemplateName"]
     class SparkConfEntry(_message.Message):
         __slots__ = ["key", "value"]
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -46,6 +47,10 @@ class SparkJob(_message.Message):
     DATABRICKSCONF_FIELD_NUMBER: _ClassVar[int]
     DATABRICKSTOKEN_FIELD_NUMBER: _ClassVar[int]
     DATABRICKSINSTANCE_FIELD_NUMBER: _ClassVar[int]
+    DRIVERPOD_FIELD_NUMBER: _ClassVar[int]
+    DRIVERPODTEMPLATENAME_FIELD_NUMBER: _ClassVar[int]
+    EXECUTORPOD_FIELD_NUMBER: _ClassVar[int]
+    EXECUTORPODTEMPLATENAME_FIELD_NUMBER: _ClassVar[int]
     applicationType: SparkApplication.Type
     mainApplicationFile: str
     mainClass: str
@@ -55,4 +60,8 @@ class SparkJob(_message.Message):
     databricksConf: _struct_pb2.Struct
     databricksToken: str
     databricksInstance: str
-    def __init__(self, applicationType: _Optional[_Union[SparkApplication.Type, str]] = ..., mainApplicationFile: _Optional[str] = ..., mainClass: _Optional[str] = ..., sparkConf: _Optional[_Mapping[str, str]] = ..., hadoopConf: _Optional[_Mapping[str, str]] = ..., executorPath: _Optional[str] = ..., databricksConf: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., databricksToken: _Optional[str] = ..., databricksInstance: _Optional[str] = ...) -> None: ...
+    driverPod: _tasks_pb2.K8sPod
+    driverPodTemplateName: str
+    executorPod: _tasks_pb2.K8sPod
+    executorPodTemplateName: str
+    def __init__(self, applicationType: _Optional[_Union[SparkApplication.Type, str]] = ..., mainApplicationFile: _Optional[str] = ..., mainClass: _Optional[str] = ..., sparkConf: _Optional[_Mapping[str, str]] = ..., hadoopConf: _Optional[_Mapping[str, str]] = ..., executorPath: _Optional[str] = ..., databricksConf: _Optional[_Union[_struct_pb2.Struct, _Mapping]] = ..., databricksToken: _Optional[str] = ..., databricksInstance: _Optional[str] = ..., driverPod: _Optional[_Union[_tasks_pb2.K8sPod, _Mapping]] = ..., driverPodTemplateName: _Optional[str] = ..., executorPod: _Optional[_Union[_tasks_pb2.K8sPod, _Mapping]] = ..., executorPodTemplateName: _Optional[str] = ...) -> None: ...
diff --git a/gen/pb_rust/flyteidl.plugins.rs b/gen/pb_rust/flyteidl.plugins.rs
index 5c7873b5d..edf7e1eab 100644
--- a/gen/pb_rust/flyteidl.plugins.rs
+++ b/gen/pb_rust/flyteidl.plugins.rs
@@ -285,6 +285,45 @@ pub struct SparkJob {
     /// This instance name can be set in either flytepropeller or flytekit.
     #[prost(string, tag="9")]
     pub databricks_instance: ::prost::alloc::string::String,
+    /// Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+    /// driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+    /// identically as, the default PodTemplate configured in FlytePropeller.
+    /// +optional
+    #[prost(string, tag="11")]
+    pub driver_pod_template_name: ::prost::alloc::string::String,
+    /// Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+    /// executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+    /// identically as, the default PodTemplate configured in FlytePropeller.
+    /// +optional
+    #[prost(string, tag="13")]
+    pub executor_pod_template_name: ::prost::alloc::string::String,
+    /// The pod spec and metadata to be used as the base configuration when creating the driver Pod for this task.
+    /// +optional
+    #[prost(oneof="spark_job::DriverPodValue", tags="10")]
+    pub driver_pod_value: ::core::option::Option,
+    /// The pod spec and metadata to be used as the base configuration when creating the executor Pods for this task.
+    /// +optional
+    #[prost(oneof="spark_job::ExecutorPodValue", tags="12")]
+    pub executor_pod_value: ::core::option::Option,
+}
+/// Nested message and enum types in `SparkJob`.
+pub mod spark_job {
+    /// The pod spec and metadata to be used as the base configuration when creating the driver Pod for this task.
+    /// +optional
+    #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum DriverPodValue {
+        #[prost(message, tag="10")]
+        DriverPod(super::super::core::K8sPod),
+    }
+    /// The pod spec and metadata to be used as the base configuration when creating the executor Pods for this task.
+    /// +optional
+    #[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum ExecutorPodValue {
+        #[prost(message, tag="12")]
+        ExecutorPod(super::super::core::K8sPod),
+    }
 }
 /// Custom proto for plugin that enables distributed training using 
 #[allow(clippy::derive_partial_eq_without_eq)]
diff --git a/protos/flyteidl/plugins/spark.proto b/protos/flyteidl/plugins/spark.proto
index 6ba00fe05..e08afb59c 100644
--- a/protos/flyteidl/plugins/spark.proto
+++ b/protos/flyteidl/plugins/spark.proto
@@ -1,8 +1,10 @@
 syntax = "proto3";
 
-package flyteidl.plugins;
+import "flyteidl/core/tasks.proto";
 import "google/protobuf/struct.proto";
 
+package flyteidl.plugins;
+
 option go_package = "github.com/flyteorg/flyteidl/gen/pb-go/flyteidl/plugins";
 
 message SparkApplication {
@@ -31,4 +33,24 @@ message SparkJob {
     // Domain name of your deployment. Use the form .cloud.databricks.com.
     // This instance name can be set in either flytepropeller or flytekit.
     string databricksInstance = 9;
+    // The pod spec and metadata to be used as the base configuration when creating the driver Pod for this task.
+    // +optional
+    oneof driverPodValue {
+        core.K8sPod driverPod = 10;
+    }
+    // Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+    // driver Pod for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+    // identically as, the default PodTemplate configured in FlytePropeller.
+    // +optional
+    string driverPodTemplateName = 11;
+    // The pod spec and metadata to be used as the base configuration when creating the executor Pods for this task.
+    // +optional
+    oneof executorPodValue {
+        core.K8sPod executorPod = 12;
+    }
+    // Reference to an existing PodTemplate k8s resource to be used as the base configuration when creating the 
+    // executor Pods for this task. If this value is set, the specified PodTemplate will be used instead of, but applied
+    // identically as, the default PodTemplate configured in FlytePropeller.
+    // +optional
+    string executorPodTemplateName = 13;
 }