[UNR-2003] Create CookAndGenerateSchemaCommandlet (#1342)

* Initial commit of CookAndGenerateSchemaCommandlet, generating correct schema but appears to be missing Data-Only Blueprints

* Wrap generate call in an async task to avoid blocking.

* New Implementation using UObjectArray callback instead, catches a lot of things missed. Also filtering UObjects to replicate based on IsSupportedForNetworking

* Avoid adding classes twice during generate. Tidy up

* Only generate schema for explicit spatial types (AActor & UActorComponent are explicit by default). Also use objectiterator with FObjectListener directly.

* Check for explicit Spatial flag, also double check parent class if no explicit flag is present as a workaround for stale blueprints

* Downgrade logs to verbose. Add Schema Gen Timer

* Cleanup

* More Cleanup

* Add back in merge error

* Update UnrealEngine Version for CI

* Remove Newline

* Fix merge issues

* PR Comments

* Use ClassIterator directly in commandlet to gather supported c++ & in-memory classes before cook

* Update unreal-engine.version

* PR Comments

* Remove recursive argument to VisitAllObjects

* Remove newline

* Remove redundant method FilterClasses

* Put Schema Gen API inside SpatialGDKEditor::Schema namespace

* Use GetAllSupportedClasses in Cook Commandlet

* Make ObjectListener interface cleaner. Sort Set directly

* Only return 0 since CookCommandlet can only return 0

* Update Engine Version in CI

* PR Comments

* Remove recursive check now that blueprint class flags are fixed in the KismetCompiler

* Update checks for NotSpatialType to include checks for ExplicitSpatialType now that semantics have changed

* Only check for the absence of SPATIALCLASS_ExplicitSpatialType since it's mutually exclusive to SPATIALCLASS_NotSpatialType

* rename SPATIALCLASS_ExplicitSpatialType SPATIALCLASS_SpatialType, remove redundant check for NotSpatialType

* Add CHANGELOG

* Increment RequiresSetup

* Better error when no generated schema for class

* Update engine-version

* Update SpatialGDK/Source/SpatialGDKEditorCommandlet/Private/Commandlets/CookAndGenerateSchemaCommandlet.cpp

Co-Authored-By: improbable-valentyn <32096431+improbable-valentyn@users.noreply.github.com>

* remove newline from engine version

* Actually remove the newline this time

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased-`x.y.z`] - 2019-xx-xx
 - Added logging for queued RPCs.
 - Added several new STAT annotations into the ServerReplicateActors call chain.
+- Avoid generating schema for all UObject subclasses. Actor, ActorComponent, GameplayAbility subclasses are enabled by default, other classes can be enabled using SpatialType UCLASS specifier.
+- Added new experimental CookAndGenerateSchemaCommandlet that generates required schema during a regular cook.
 
 ### Features:
 - Visual Studio 2019 is now supported.

RequireSetup

@@ -1,4 +1,4 @@
 Increment the below number whenever it is required to run Setup.bat as part of a new commit.
 Our git hooks will detect this file has been updated and automatically run Setup.bat on pull.
 
-32
+33

SpatialGDK/Source/SpatialGDK/Private/EngineClasses/SpatialNetDriver.cpp

@@ -998,7 +998,7 @@ void USpatialNetDriver::ServerReplicateActors_ProcessPrioritizedActors(UNetConne
 				// or it's an editor placed actor and the client hasn't initialized the level it's in
 				if (Channel == nullptr && GuidCache->SupportsObject(Actor->GetClass()) && GuidCache->SupportsObject(Actor->IsNetStartupActor() ? Actor : Actor->GetArchetype()))
 				{
-					if (Actor->GetClass()->HasAnySpatialClassFlags(SPATIALCLASS_NotSpatialType))
+					if (!Actor->GetClass()->HasAnySpatialClassFlags(SPATIALCLASS_SpatialType))
 					{
 						// Trying to replicate an actor that isn't supported by Spatial (e.g. marked NotSpatial)
 						continue;
@@ -1301,7 +1301,7 @@ void USpatialNetDriver::ProcessRemoteFunction(
 	// The RPC might have been called by an actor directly, or by a subobject on that actor
 	UObject* CallingObject = SubObject ? SubObject : Actor;
 
-	if (CallingObject->GetClass()->HasAnySpatialClassFlags(SPATIALCLASS_NotSpatialType))
+	if (!CallingObject->GetClass()->HasAnySpatialClassFlags(SPATIALCLASS_SpatialType))
 	{
 		UE_LOG(LogSpatialOSNetDriver, Verbose, TEXT("Trying to call RPC %s on object %s (class %s) that isn't supported by Spatial. This RPC will be dropped."), *Function->GetName(), *CallingObject->GetName(), *CallingObject->GetClass()->GetName());
 		return;

SpatialGDK/Source/SpatialGDK/Private/Interop/SpatialClassInfoManager.cpp

@@ -100,7 +100,7 @@ void USpatialClassInfoManager::CreateClassInfoForClass(UClass* Class)
 	// Note: we have to add Class to ClassInfoMap before quitting, as it is expected to be in there by GetOrCreateClassInfoByClass. Therefore the quitting logic cannot be moved higher up.
 	if (!IsSupportedClass(ClassPath))
 	{
-		UE_LOG(LogSpatialClassInfoManager, Error, TEXT("Could not find class %s in schema database. Double-check whether replication is enabled for this class, the class is explicitly referenced from the starting scene and schema has been generated."), *ClassPath);
+		UE_LOG(LogSpatialClassInfoManager, Error, TEXT("Could not find class %s in schema database. Double-check whether replication is enabled for this class, the class is marked as SpatialType, and schema has been generated."), *ClassPath);
 		UE_LOG(LogSpatialClassInfoManager, Error, TEXT("Disconnecting due to no generated schema for %s."), *ClassPath);
 		QuitGame();
 		return;

SpatialGDK/Source/SpatialGDK/Private/Utils/InterestFactory.cpp

@@ -35,7 +35,11 @@ void GatherClientInterestDistances()
 	TMap<UClass*, float> DiscoveredInterestDistancesSquared;
 	for (TObjectIterator<UClass> It; It; ++It)
 	{
-		if (It->HasAnySpatialClassFlags(SPATIALCLASS_ServerOnly | SPATIALCLASS_NotSpatialType))
+		if (It->HasAnySpatialClassFlags(SPATIALCLASS_ServerOnly))
+		{
+			continue;
+		}
+		if (!It->HasAnySpatialClassFlags(SPATIALCLASS_SpatialType))
 		{
 			continue;
 		}

SpatialGDK/Source/SpatialGDK/Public/Utils/EngineVersionCheck.h

@@ -7,7 +7,7 @@
 // GDK Version to be updated with SPATIAL_ENGINE_VERSION
 // when breaking changes are made to the engine that requires
 // changes to the GDK to remain compatible
-#define SPATIAL_GDK_VERSION 6
+#define SPATIAL_GDK_VERSION 7
 
 // Check if GDK is compatible with the current version of Unreal Engine
 // SPATIAL_ENGINE_VERSION is incremented in engine when breaking changes

SpatialGDK/Source/SpatialGDKEditor/Private/SchemaGenerator/SchemaGenerator.cpp

@@ -12,6 +12,9 @@
 #include "Utils/CodeWriter.h"
 #include "Utils/ComponentIdGenerator.h"
 #include "Utils/DataTypeUtilities.h"
+#include "SpatialGDKEditorSchemaGenerator.h"
+
+using namespace SpatialGDKEditor::Schema;
 
 DEFINE_LOG_CATEGORY(LogSchemaGenerator);
 
@@ -565,7 +568,7 @@ void GenerateSubobjectSchemaForActorIncludes(FCodeWriter& Writer, TSharedPtr<FUn
 		{
 			UObject* Value = PropertyTypeInfo->Object;
 
-			if (Value != nullptr && !Value->IsEditorOnly())
+			if (Value != nullptr && IsSupportedClass(Value->GetClass()))
 			{
 				UClass* Class = Value->GetClass();
 				if (!AlreadyImported.Contains(Class) && SchemaGeneratedClasses.Contains(Class))

SpatialGDK/Source/SpatialGDKEditor/Private/SchemaGenerator/SpatialGDKEditorSchemaGenerator.cpp

@@ -57,7 +57,9 @@ const FString SchemaDatabasePackagePath = FPaths::Combine(FPaths::ProjectContent
 const FString SchemaDatabaseAssetPath = FPaths::SetExtension(SpatialConstants::SCHEMA_DATABASE_ASSET_PATH, TEXT(".SchemaDatabase"));
 const FString SchemaDatabaseFileName = FPaths::SetExtension(SchemaDatabasePackagePath, FPackageName::GetAssetPackageExtension());
 
-namespace
+namespace SpatialGDKEditor
+{
+namespace Schema
 {
 
 void AddPotentialNameCollision(const FString& DesiredSchemaName, const FString& ClassPath, const FString& GeneratedSchemaName)
@@ -240,8 +242,6 @@ bool ValidateIdentifierNames(TArray<TSharedPtr<FUnrealType>>& TypeInfos)
 	return bSuccess;
 }
 
-}// ::
-
 void GenerateSchemaFromClasses(const TArray<TSharedPtr<FUnrealType>>& TypeInfos, const FString& CombinedSchemaPath, FComponentIdGenerator& IdGenerator)
 {
 	// Generate the actual schema.
@@ -410,47 +410,78 @@ bool SaveSchemaDatabase()
 	return true;
 }
 
-TArray<UClass*> GetAllSupportedClasses()
+bool IsSupportedClass(const UClass* SupportedClass)
 {
-	TSet<UClass*> Classes;
+	if (!IsValid(SupportedClass))
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Invalid Class not supported for schema gen."), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
+
+	if (SupportedClass->IsEditorOnly())
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Editor-only Class not supported for schema gen."), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
+
+	if (!SupportedClass->HasAnySpatialClassFlags(SPATIALCLASS_SpatialType))
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] No SpatialType flag, not supported for schema gen."), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
+
+	if (SupportedClass->HasAnyClassFlags(CLASS_LayoutChanging))
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Layout changing, not supported"), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
+
+	// Ensure we don't process transient generated classes for BP
+	if (SupportedClass->GetName().StartsWith(TEXT("SKEL_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("REINST_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("TRASHCLASS_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("HOTRELOADED_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("PROTO_BP_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("PLACEHOLDER-CLASS_"), ESearchCase::CaseSensitive)
+		|| SupportedClass->GetName().StartsWith(TEXT("ORPHANED_DATA_ONLY_"), ESearchCase::CaseSensitive))
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Transient Class not supported for schema gen"), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
+
 	const TArray<FDirectoryPath>& DirectoriesToNeverCook = GetDefault<UProjectPackagingSettings>()->DirectoriesToNeverCook;
 
-	for (TObjectIterator<UClass> ClassIt; ClassIt; ++ClassIt)
+	// Avoid processing classes contained in Directories to Never Cook
+	const FString& ClassPath = SupportedClass->GetPathName();
+	if (DirectoriesToNeverCook.ContainsByPredicate([&ClassPath](const FDirectoryPath& Directory)
 	{
-		// User told us to ignore this class
-		if (ClassIt->HasAnySpatialClassFlags(SPATIALCLASS_NotSpatialType))
-		{
-			continue;
-		}
+		return ClassPath.StartsWith(Directory.Path);
+	}))
+	{
+		UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Inside Directory to never cook for schema gen"), *GetPathNameSafe(SupportedClass));
+		return false;
+	}
 
-		UClass* SupportedClass = *ClassIt;
+	UE_LOG(LogSpatialGDKSchemaGenerator, Verbose, TEXT("[%s] Supported Class"), *GetPathNameSafe(SupportedClass));
+	return true;
+}
 
-		// Ensure we don't process transient generated classes for BP
-		if (SupportedClass->GetName().StartsWith(TEXT("SKEL_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("REINST_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("TRASHCLASS_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("HOTRELOADED_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("PROTO_BP_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("PLACEHOLDER-CLASS_"), ESearchCase::CaseSensitive)
-			|| SupportedClass->GetName().StartsWith(TEXT("ORPHANED_DATA_ONLY_"), ESearchCase::CaseSensitive))
-		{
-			continue;
-		}
 
-		// Avoid processing classes contained in Directories to Never Cook
-		const FString& ClassPath = SupportedClass->GetPathName();
-		if (DirectoriesToNeverCook.ContainsByPredicate([&ClassPath](const FDirectoryPath& Directory)
-		{
-			return ClassPath.StartsWith(Directory.Path);
-		}))
+TSet<UClass*> GetAllSupportedClasses()
+{
+	TSet<UClass*> Classes;
+
+	for (TObjectIterator<UClass> ClassIt; ClassIt; ++ClassIt)
+	{
+		UClass* SupportedClass = *ClassIt;
+
+		if (IsSupportedClass(SupportedClass))
 		{
-			continue;
+			Classes.Add(SupportedClass);
 		}
-
-		Classes.Add(SupportedClass);
 	}
 
-	return Classes.Array();
+	return Classes;
 }
 
 void CopyWellKnownSchemaFiles()
@@ -560,7 +591,7 @@ bool TryLoadExistingSchemaDatabase()
 	return true;
 }
 
-SPATIALGDKEDITOR_API bool GeneratedSchemaFolderExists()
+bool GeneratedSchemaFolderExists()
 {
 	const FString SchemaOutputPath = GetDefault<USpatialGDKEditorSettings>()->GetGeneratedSchemaOutputFolder();
 	IPlatformFile& PlatformFile = FPlatformFileManager::Get().GetPlatformFile();
@@ -686,19 +717,55 @@ bool RunSchemaCompiler()
 
 bool SpatialGDKGenerateSchema()
 {
-	ResetUsedNames();
+	// Generate Schema for classes loaded in memory.
+
+	if (!SpatialGDKGenerateSchemaForClasses(GetAllSupportedClasses()))
+	{
+		return false;
+	}
+
+	if (!SaveSchemaDatabase())
+	{
+		return false;
+	}
 
-	// Gets the classes currently loaded into memory.
-	SchemaGeneratedClasses = GetAllSupportedClasses();
-	SchemaGeneratedClasses.Sort();
+	return RunSchemaCompiler();
+}
 
+bool SpatialGDKGenerateSchemaForClasses(TSet<UClass*> Classes)
+{
+	ResetUsedNames();
+	Classes.Sort([](const UClass& A, const UClass& B)
+	{
+		return A.GetPathName() < B.GetPathName();
+	});
+
 	// Generate Type Info structs for all classes
 	TArray<TSharedPtr<FUnrealType>> TypeInfos;
 
-	for (const auto& Class : SchemaGeneratedClasses)
+	for (const auto& Class : Classes)
 	{
+		if (SchemaGeneratedClasses.Contains(Class))
+		{
+			continue;
+		}
+
+		SchemaGeneratedClasses.Add(Class);
 		// Parent and static array index start at 0 for checksum calculations.
-		TypeInfos.Add(CreateUnrealTypeInfo(Class, 0, 0));
+		TSharedPtr<FUnrealType> TypeInfo = CreateUnrealTypeInfo(Class, 0, 0);
+		TypeInfos.Add(TypeInfo);
+		VisitAllObjects(TypeInfo, [&](TSharedPtr<FUnrealType> TypeNode)
+		{
+			if (UClass* NestedClass = Cast<UClass>(TypeNode->Type))
+			{
+				if (!SchemaGeneratedClasses.Contains(NestedClass) && IsSupportedClass(NestedClass))
+				{
+					TypeInfos.Add(CreateUnrealTypeInfo(NestedClass, 0, 0));
+					SchemaGeneratedClasses.Add(NestedClass);
+				}
+			}
+			return true;
+		});
 	}
 
 	if (!ValidateIdentifierNames(TypeInfos))
@@ -724,12 +791,11 @@ bool SpatialGDKGenerateSchema()
 	GenerateSchemaFromClasses(TypeInfos, SchemaOutputPath, IdGenerator);
 	GenerateSchemaForSublevels(SchemaOutputPath, IdGenerator);
 	NextAvailableComponentId = IdGenerator.Peek();
-	if (!SaveSchemaDatabase())
-	{
-		return false;
-	}
 
-	return RunSchemaCompiler();
+	return true;
 }
 
+} // Schema
+} // SpatialGDKEditor
+
 #undef LOCTEXT_NAMESPACE

SpatialGDK/Source/SpatialGDKEditor/Private/SchemaGenerator/TypeStructure.cpp

@@ -7,11 +7,7 @@
 #include "SpatialGDKEditorSchemaGenerator.h"
 #include "Utils/RepLayoutUtils.h"
 
-namespace Errors
-{
-	FString DuplicateComponentError = TEXT("WARNING: Unreal GDK does not currently support multiple static components of the same type.\n"
-		"Make sure {0} has only one instance of {1} or don't generate type bindings for {2}");
-}
+using namespace SpatialGDKEditor::Schema;
 
 TArray<EReplicatedPropertyGroup> GetAllReplicatedPropertyGroups()
 {
@@ -24,6 +20,19 @@ FString GetReplicatedPropertyGroupName(EReplicatedPropertyGroup Group)
 	return Group == REP_SingleClient ? TEXT("OwnerOnly") : TEXT("");
 }
 
+void VisitAllObjects(TSharedPtr<FUnrealType> TypeNode, TFunction<bool(TSharedPtr<FUnrealType>)> Visitor)
+{
+	bool bShouldRecurseFurther = Visitor(TypeNode);
+	for (auto& PropertyPair : TypeNode->Properties)
+	{
+		if (bShouldRecurseFurther && PropertyPair.Value->Type.IsValid())
+		{
+			// Recurse into subobjects.
+			VisitAllObjects(PropertyPair.Value->Type, Visitor);
+		}
+	}
+}
+
 void VisitAllProperties(TSharedPtr<FUnrealType> TypeNode, TFunction<bool(TSharedPtr<FUnrealProperty>)> Visitor)
 {
 	for (auto& PropertyPair : TypeNode->Properties)
@@ -465,7 +474,7 @@ FSubobjectMap GetAllSubobjects(TSharedPtr<FUnrealType> TypeInfo)
 		{
 			UObject* Value = PropertyTypeInfo->Object;
 
-			if (Value != nullptr && !Value->IsEditorOnly())
+			if (Value != nullptr && IsSupportedClass(Value->GetClass()))
 			{
 				if (!SeenComponents.Contains(Value))
 				{

SpatialGDK/Source/SpatialGDKEditor/Private/SchemaGenerator/TypeStructure.h

@@ -123,6 +123,11 @@ TArray<EReplicatedPropertyGroup> GetAllReplicatedPropertyGroups();
 // Convert a replicated property group to a string. Used to generate component names.
 FString GetReplicatedPropertyGroupName(EReplicatedPropertyGroup Group);
 
+// Given an AST, this applies the function 'Visitor' to all FUnrealType's contained transitively within the properties. bRecurseIntoObjects will control	
+// whether this function will recurse into a UObject's properties, which may not always be desirable. However, it will always recurse into substructs.	
+// If the Visitor function returns false, it will not recurse any further into that part of the tree.
+void VisitAllObjects(TSharedPtr<FUnrealType> TypeNode, TFunction<bool(TSharedPtr<FUnrealType>)> Visitor);
+
 // Given an AST, this applies the function 'Visitor' to all properties contained transitively within the type. This will recurse into substructs.
 // If the Visitor function returns false, it will not recurse any further into that part of the tree.
 void VisitAllProperties(TSharedPtr<FUnrealType> TypeNode, TFunction<bool(TSharedPtr<FUnrealProperty>)> Visitor);

SpatialGDK/Source/SpatialGDKEditor/Private/SchemaGenerator/Utils/CodeWriter.cpp

@@ -120,4 +120,4 @@ void FCodeWriter::WriteToFile(const FString& Filename)
 void FCodeWriter::Dump()
 {
 	UE_LOG(LogTemp, Warning, TEXT("%s"), *OutputSource);
-}
+}