AssetCurator.h

#pragma once
 
#include <Core/Configuration/PlatformProfile.h>
#include <EditorFramework/Assets/AssetDocumentInfo.h>
#include <EditorFramework/Assets/AssetDocumentManager.h>
#include <EditorFramework/Assets/Declarations.h>
#include <EditorFramework/EditorFrameworkDLL.h>
#include <Foundation/Algorithm/HashHelperString.h>
#include <Foundation/Application/Config/FileSystemConfig.h>
#include <Foundation/Configuration/Singleton.h>
#include <Foundation/Containers/HashTable.h>
#include <Foundation/IO/DirectoryWatcher.h>
#include <Foundation/Logging/LogEntry.h>
#include <Foundation/Profiling/Profiling.h>
#include <Foundation/Threading/AtomicInteger.h>
#include <Foundation/Threading/DelegateTask.h>
#include <Foundation/Threading/LockedObject.h>
#include <Foundation/Threading/Mutex.h>
#include <Foundation/Threading/TaskSystem.h>
#include <Foundation/Time/Timestamp.h>
#include <ToolsFoundation/Document/DocumentManager.h>
#include <ToolsFoundation/FileSystem/DataDirPath.h>
#include <ToolsFoundation/FileSystem/Declarations.h>
 
#include <tuple>
 
class plUpdateTask;
class plTask;
class plAssetDocumentManager;
class plDirectoryWatcher;
class plProcessTask;
struct plFileStats;
class plAssetProcessorLog;
class plFileSystemWatcher;
class plAssetTableWriter;
struct plFileChangedEvent;
class plFileSystemModel;
 
 
#if 0 // Define to enable extensive curator profile scopes
#  define CURATOR_PROFILE(szName) PL_PROFILE_SCOPE(szName)
 
#else
#  define CURATOR_PROFILE(Name)
 
#endif
 
class plCuratorMutex : public plMutex
{
public:
  void Lock()
  {
    CURATOR_PROFILE("plCuratorMutex");
    plMutex::Lock();
  }
 
  void Unlock() { plMutex::Unlock(); }
};
 
struct PL_EDITORFRAMEWORK_DLL plAssetInfo
{
  plAssetInfo() = default;
  void Update(plUniquePtr<plAssetInfo>& rhs);
 
  plAssetDocumentManager* GetManager() { return static_cast<plAssetDocumentManager*>(m_pDocumentTypeDescriptor->m_pManager); }
 
  enum TransformState : plUInt8
  {
    Unknown = 0,
    UpToDate,
    NeedsImport,
    NeedsTransform,
    NeedsThumbnail,
    TransformError,
    MissingTransformDependency,
    MissingThumbnailDependency,
    CircularDependency,
    COUNT,
  };
 
  plUInt8 m_LastStateUpdate = 0; 
  plAssetExistanceState::Enum m_ExistanceState = plAssetExistanceState::FileAdded;
  TransformState m_TransformState = TransformState::Unknown;
  plUInt64 m_AssetHash = 0; 
  plUInt64 m_ThumbHash = 0; 
 
  plDynamicArray<plLogEntry> m_LogEntries;
 
  const plAssetDocumentTypeDescriptor* m_pDocumentTypeDescriptor = nullptr;
  plDataDirPath m_Path;
 
  plUniquePtr<plAssetDocumentInfo> m_Info;
 
  plSet<plString> m_MissingTransformDeps;
  plSet<plString> m_MissingThumbnailDeps;
  plSet<plString> m_CircularDependencies;
 
  plSet<plUuid> m_SubAssets; 
 
private:
  PL_DISALLOW_COPY_AND_ASSIGN(plAssetInfo);
};
 
struct PL_EDITORFRAMEWORK_DLL plSubAsset
{
  plStringView GetName() const;
  void GetSubAssetIdentifier(plStringBuilder& out_sPath) const;
 
  plAssetExistanceState::Enum m_ExistanceState = plAssetExistanceState::FileAdded;
  plAssetInfo* m_pAssetInfo = nullptr;
  plTime m_LastAccess;
  bool m_bMainAsset = true;
 
  plSubAssetData m_Data;
};
 
 
 
struct plAssetCuratorEvent
{
  enum class Type
  {
    AssetAdded,
    AssetRemoved,
    AssetMoved,
    AssetUpdated,
    AssetListReset,
    ActivePlatformChanged,
  };
 
  plUuid m_AssetGuid;
  const plSubAsset* m_pInfo;
  Type m_Type;
};
 
class PL_EDITORFRAMEWORK_DLL plAssetCurator
{
  PL_DECLARE_SINGLETON(plAssetCurator);
 
public:
  plAssetCurator();
  ~plAssetCurator();
 
 
  void StartInitialize(const plApplicationFileSystemConfig& cfg);
  void WaitForInitialize();
  void Deinitialize();
 
  void MainThreadTick(bool bTopLevel);
 
 
public:
  const plPlatformProfile* GetDevelopmentAssetProfile() const;
 
  const plPlatformProfile* GetActiveAssetProfile() const;
 
  plUInt32 GetActiveAssetProfileIndex() const;
 
  plUInt32 FindAssetProfileByName(const char* szPlatform);
 
  plUInt32 GetNumAssetProfiles() const;
 
  const plPlatformProfile* GetAssetProfile(plUInt32 uiIndex) const;
 
  plPlatformProfile* GetAssetProfile(plUInt32 uiIndex);
 
  plPlatformProfile* CreateAssetProfile();
 
  plResult DeleteAssetProfile(plPlatformProfile* pProfile);
 
  void SetActiveAssetProfileByIndex(plUInt32 uiIndex, bool bForceReevaluation = false);
 
  plResult SaveAssetProfiles();
 
  void SaveRuntimeProfiles();
 
private:
  void ClearAssetProfiles();
  void SetupDefaultAssetProfiles();
  plResult LoadAssetProfiles();
  void ComputeAllDocumentManagerAssetProfileHashes();
 
  plHybridArray<plPlatformProfile*, 8> m_AssetProfiles;
 
 
public:
  plDateTime GetLastFullTransformDate() const;
  void StoreFullTransformDate();
 
  plStatus TransformAllAssets(plBitflags<plTransformFlags> transformFlags, const plPlatformProfile* pAssetProfile = nullptr);
  void ResaveAllAssets();
  plTransformStatus TransformAsset(const plUuid& assetGuid, plBitflags<plTransformFlags> transformFlags, const plPlatformProfile* pAssetProfile = nullptr);
  plTransformStatus CreateThumbnail(const plUuid& assetGuid);
 
  void TransformAssetsForSceneExport(const plPlatformProfile* pAssetProfile = nullptr);
 
  plResult WriteAssetTables(const plPlatformProfile* pAssetProfile = nullptr, bool bForce = false);
 
  using plLockedSubAsset = plLockedObject<plMutex, const plSubAsset>;
 
  const plLockedSubAsset FindSubAsset(plStringView sPathOrGuid, bool bExhaustiveSearch = false) const;
 
  const plLockedSubAsset GetSubAsset(const plUuid& assetGuid) const;
 
  using plLockedSubAssetTable = plLockedObject<plMutex, const plHashTable<plUuid, plSubAsset>>;
 
  const plLockedSubAssetTable GetKnownSubAssets() const;
 
  using plLockedAssetTable = plLockedObject<plMutex, const plHashTable<plUuid, plAssetInfo*>>;
 
  const plLockedAssetTable GetKnownAssets() const;
 
  plUInt64 GetAssetDependencyHash(plUuid assetGuid);
 
  plUInt64 GetAssetReferenceHash(plUuid assetGuid);
 
  plAssetInfo::TransformState IsAssetUpToDate(const plUuid& assetGuid, const plPlatformProfile* pAssetProfile, const plAssetDocumentTypeDescriptor* pTypeDescriptor, plUInt64& out_uiAssetHash, plUInt64& out_uiThumbHash, bool bForce = false);
  void GetAssetTransformStats(plUInt32& out_uiNumAssets, plHybridArray<plUInt32, plAssetInfo::TransformState::COUNT>& out_count);
 
  plString FindDataDirectoryForAsset(plStringView sAbsoluteAssetPath) const;
 
  plResult FindBestMatchForFile(plStringBuilder& ref_sFile, plArrayPtr<plString> allowedFileExtensions) const;
 
  void FindAllUses(plUuid assetGuid, plSet<plUuid>& ref_uses, bool bTransitive) const;
 
  void FindAllUses(plStringView sAbsolutePath, plSet<plUuid>& ref_uses) const;
 
  bool IsReferenced(plStringView sAbsolutePath) const;
 
 
 
  void NotifyOfFileChange(plStringView sAbsolutePath);
  void NotifyOfAssetChange(const plUuid& assetGuid);
  void UpdateAssetLastAccessTime(const plUuid& assetGuid);
 
  void CheckFileSystem();
 
  void NeedsReloadResources(const plUuid& assetGuid);
 
  void InvalidateAssetsWithTransformState(plAssetInfo::TransformState state);
 
 
 
 
  void GenerateTransitiveHull(const plStringView sAssetOrPath, plSet<plString>& inout_deps, bool bIncludeTransformDeps = false, bool bIncludeThumbnailDeps = false, bool bIncludePackageDeps = false) const;
 
  void GenerateInverseTransitiveHull(const plAssetInfo* pAssetInfo, plSet<plUuid>& inout_inverseDeps, bool bIncludeTransformDeps = false, bool bIncludeThumbnailDeps = false) const;
 
  void WriteDependencyDGML(const plUuid& guid, plStringView sOutputFile) const;
 
 
public:
  plEvent<const plAssetCuratorEvent&> m_Events;
 
private:
 
  plTransformStatus ProcessAsset(plAssetInfo* pAssetInfo, const plPlatformProfile* pAssetProfile, plBitflags<plTransformFlags> transformFlags);
  plStatus ResaveAsset(plAssetInfo* pAssetInfo);
  plAssetInfo* GetAssetInfo(const plUuid& assetGuid);
  const plAssetInfo* GetAssetInfo(const plUuid& assetGuid) const;
 
  plSubAsset* GetSubAssetInternal(const plUuid& assetGuid);
 
  plAssetInfo* GetAssetInfo(const plString& sAssetGuid);
 
  void OnFileChangedEvent(const plFileChangedEvent& e);
 
  void ProcessAllCoreAssets();
 
 
  void RestartUpdateTask();
  void ShutdownUpdateTask();
 
  bool GetNextAssetToUpdate(plUuid& out_guid, plStringBuilder& out_sAbsPath);
  void OnUpdateTaskFinished(const plSharedPtr<plTask>& pTask);
  void RunNextUpdateTask();
 
 
  plAssetInfo::TransformState HashAsset(
    plUInt64 uiSettingsHash, const plHybridArray<plString, 16>& assetTransformDeps, const plHybridArray<plString, 16>& assetThumbnailDeps, plSet<plString>& missingTransformDeps, plSet<plString>& missingThumbnailDeps, plUInt64& out_AssetHash, plUInt64& out_ThumbHash, bool bForce);
  bool AddAssetHash(plString& sPath, bool bIsReference, plUInt64& out_AssetHash, plUInt64& out_ThumbHash, bool bForce);
 
  plResult EnsureAssetInfoUpdated(const plDataDirPath& absFilePath, const plFileStatus& stat, bool bForce = false);
  void TrackDependencies(plAssetInfo* pAssetInfo);
  void UntrackDependencies(plAssetInfo* pAssetInfo);
  plResult CheckForCircularDependencies(plAssetInfo* pAssetInfo);
  void UpdateTrackedFiles(const plUuid& assetGuid, const plSet<plString>& files, plMap<plString, plHybridArray<plUuid, 1>>& inverseTracker, plSet<std::tuple<plUuid, plUuid>>& unresolved, bool bAdd);
  void UpdateUnresolvedTrackedFiles(plMap<plString, plHybridArray<plUuid, 1>>& inverseTracker, plSet<std::tuple<plUuid, plUuid>>& unresolved);
  plResult ReadAssetDocumentInfo(const plDataDirPath& absFilePath, const plFileStatus& stat, plUniquePtr<plAssetInfo>& assetInfo);
  void UpdateSubAssets(plAssetInfo& assetInfo);
 
  void RemoveAssetTransformState(const plUuid& assetGuid);
  void InvalidateAssetTransformState(const plUuid& assetGuid);
 
  plAssetInfo::TransformState UpdateAssetTransformState(plUuid assetGuid, plUInt64& out_AssetHash, plUInt64& out_ThumbHash, bool bForce);
  void UpdateAssetTransformState(const plUuid& assetGuid, plAssetInfo::TransformState state);
  void UpdateAssetTransformLog(const plUuid& assetGuid, plDynamicArray<plLogEntry>& logEntries);
  void SetAssetExistanceState(plAssetInfo& assetInfo, plAssetExistanceState::Enum state);
 
  void SetAllAssetStatusUnknown();
  void LoadCaches(plMap<plDataDirPath, plFileStatus, plCompareDataDirPath>& out_referencedFiles, plMap<plDataDirPath, plFileStatus::Status, plCompareDataDirPath>& out_referencedFolders);
  void SaveCaches(const plMap<plDataDirPath, plFileStatus, plCompareDataDirPath>& referencedFiles, const plMap<plDataDirPath, plFileStatus::Status, plCompareDataDirPath>& referencedFolders);
  static void BuildFileExtensionSet(plSet<plString>& AllExtensions);
 
 
public:
  void ClearAssetCaches(plAssetDocumentManager::OutputReliability threshold);
 
 
private:
  friend class plUpdateTask;
  friend class plAssetProcessor;
  friend class plProcessTask;
 
  mutable plCuratorMutex m_CuratorMutex; // Global lock
  plTaskGroupID m_InitializeCuratorTaskID;
 
  plUInt32 m_uiActiveAssetProfile = 0;
 
  // Actual data stored in the curator
  plHashTable<plUuid, plAssetInfo*> m_KnownAssets;
  plHashTable<plUuid, plSubAsset> m_KnownSubAssets;
 
  // Derived dependency lookup tables
  plMap<plString, plHybridArray<plUuid, 1>> m_InverseTransformDeps; // [Absolute path -> asset Guid]
  plMap<plString, plHybridArray<plUuid, 1>> m_InverseThumbnailDeps; // [Absolute path -> asset Guid]
  plSet<std::tuple<plUuid, plUuid>> m_UnresolvedTransformDeps;      
  plSet<std::tuple<plUuid, plUuid>> m_UnresolvedThumbnailDeps;
 
  // State caches
  plHashSet<plUuid> m_TransformState[plAssetInfo::TransformState::COUNT];
  plHashSet<plUuid> m_SubAssetChanged; 
  plHashSet<plUuid> m_TransformStateStale;
  plHashSet<plUuid> m_Updating;
 
  // Serialized cache
  mutable plCuratorMutex m_CachedAssetsMutex; 
  plMap<plString, plUniquePtr<plAssetDocumentInfo>> m_CachedAssets;
  plMap<plString, plFileStatus> m_CachedFiles;
 
  // Immutable data after StartInitialize
  plApplicationFileSystemConfig m_FileSystemConfig;
  plUniquePtr<plAssetTableWriter> m_pAssetTableWriter;
  plSet<plString> m_ValidAssetExtensions;
 
  // Update task
  bool m_bRunUpdateTask = false;
  plSharedPtr<plUpdateTask> m_pUpdateTask;
  plTaskGroupID m_UpdateTaskGroup;
};
 
class plUpdateTask final : public plTask
{
public:
  plUpdateTask(plOnTaskFinishedCallback onTaskFinished);
  ~plUpdateTask();
 
private:
  plStringBuilder m_sAssetPath;
 
  virtual void Execute() override;
};