From 15740faf9fe9fe4be08965098bbf2947e096aeeb Mon Sep 17 00:00:00 2001 From: chai Date: Wed, 14 Aug 2019 22:50:43 +0800 Subject: +Unity Runtime code --- Runtime/Serialize/AwakeFromLoadQueue.cpp | 395 ++++ Runtime/Serialize/AwakeFromLoadQueue.h | 83 + Runtime/Serialize/Blobification/BlobSize.h | 161 ++ Runtime/Serialize/Blobification/BlobTests.cpp | 358 +++ Runtime/Serialize/Blobification/BlobWrite.cpp | 148 ++ Runtime/Serialize/Blobification/BlobWrite.h | 220 ++ .../Blobification/BlobWriteTargetSupport.cpp | 31 + .../Blobification/BlobWriteTargetSupport.h | 6 + Runtime/Serialize/Blobification/OffsetPtrTest.cpp | 31 + Runtime/Serialize/Blobification/ReduceCopyData.h | 8 + Runtime/Serialize/Blobification/offsetptr.h | 257 +++ Runtime/Serialize/BuildTargetVerification.h | 78 + Runtime/Serialize/CacheWrap.cpp | 461 ++++ Runtime/Serialize/CacheWrap.h | 229 ++ Runtime/Serialize/DumpSerializedDataToText.cpp | 372 ++++ Runtime/Serialize/DumpSerializedDataToText.h | 24 + Runtime/Serialize/FileCache.cpp | 461 ++++ Runtime/Serialize/FileCache.h | 360 +++ Runtime/Serialize/FloatStringConversion.cpp | 80 + Runtime/Serialize/FloatStringConversion.h | 16 + Runtime/Serialize/IterateTypeTree.h | 155 ++ Runtime/Serialize/LoadProgress.h | 24 + Runtime/Serialize/PathNamePersistentManager.cpp | 51 + Runtime/Serialize/PathNamePersistentManager.h | 31 + Runtime/Serialize/PersistentManager.cpp | 2291 ++++++++++++++++++++ Runtime/Serialize/PersistentManager.h | 473 ++++ Runtime/Serialize/Remapper.h | 297 +++ Runtime/Serialize/SerializationMetaFlags.h | 292 +++ Runtime/Serialize/SerializationTests.cpp | 242 +++ Runtime/Serialize/SerializeConversion.h | 34 + Runtime/Serialize/SerializeTraits.h | 533 +++++ Runtime/Serialize/SerializeTraitsBase.h | 61 + Runtime/Serialize/SerializeUtility.h | 95 + Runtime/Serialize/SerializedFile.cpp | 1520 +++++++++++++ Runtime/Serialize/SerializedFile.h | 264 +++ Runtime/Serialize/SerializedFileTests.cpp | 39 + Runtime/Serialize/SwapEndianArray.h | 27 + Runtime/Serialize/SwapEndianBytes.h | 89 + Runtime/Serialize/TransferFunctionFwd.h | 14 + .../Serialize/TransferFunctions/ProxyTransfer.cpp | 250 +++ .../Serialize/TransferFunctions/ProxyTransfer.h | 159 ++ .../TransferFunctions/ProxyTransferTests.cpp | 216 ++ .../TransferFunctions/RemapPPtrTransfer.cpp | 31 + .../TransferFunctions/RemapPPtrTransfer.h | 178 ++ .../TransferFunctions/RemapPPtrTransferTests.cpp | 57 + .../Serialize/TransferFunctions/SafeBinaryRead.cpp | 487 +++++ .../Serialize/TransferFunctions/SafeBinaryRead.h | 290 +++ .../TransferFunctions/SerializeTransfer.h | 14 + .../TransferFunctions/StreamedBinaryRead.cpp | 54 + .../TransferFunctions/StreamedBinaryRead.h | 174 ++ .../TransferFunctions/StreamedBinaryWrite.cpp | 76 + .../TransferFunctions/StreamedBinaryWrite.h | 187 ++ Runtime/Serialize/TransferFunctions/TransferBase.h | 177 ++ .../TransferFunctions/TransferNameConversions.cpp | 76 + .../TransferFunctions/TransferNameConversions.h | 56 + Runtime/Serialize/TransferFunctions/YAMLRead.cpp | 238 ++ Runtime/Serialize/TransferFunctions/YAMLRead.h | 422 ++++ .../TransferFunctions/YAMLSerializeTraits.cpp | 139 ++ .../TransferFunctions/YAMLSerializeTraits.h | 236 ++ Runtime/Serialize/TransferFunctions/YAMLWrite.cpp | 183 ++ Runtime/Serialize/TransferFunctions/YAMLWrite.h | 340 +++ .../Serialize/TransferFunctions/YAMLWriteTests.cpp | 58 + Runtime/Serialize/TransferUtility.cpp | 461 ++++ Runtime/Serialize/TransferUtility.h | 100 + Runtime/Serialize/TypeTree.cpp | 531 +++++ Runtime/Serialize/TypeTree.h | 130 ++ Runtime/Serialize/WriteData.h | 27 + Runtime/Serialize/WriteTypeToBuffer.h | 21 + 68 files changed, 15679 insertions(+) create mode 100644 Runtime/Serialize/AwakeFromLoadQueue.cpp create mode 100644 Runtime/Serialize/AwakeFromLoadQueue.h create mode 100644 Runtime/Serialize/Blobification/BlobSize.h create mode 100644 Runtime/Serialize/Blobification/BlobTests.cpp create mode 100644 Runtime/Serialize/Blobification/BlobWrite.cpp create mode 100644 Runtime/Serialize/Blobification/BlobWrite.h create mode 100644 Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp create mode 100644 Runtime/Serialize/Blobification/BlobWriteTargetSupport.h create mode 100644 Runtime/Serialize/Blobification/OffsetPtrTest.cpp create mode 100644 Runtime/Serialize/Blobification/ReduceCopyData.h create mode 100644 Runtime/Serialize/Blobification/offsetptr.h create mode 100644 Runtime/Serialize/BuildTargetVerification.h create mode 100644 Runtime/Serialize/CacheWrap.cpp create mode 100644 Runtime/Serialize/CacheWrap.h create mode 100644 Runtime/Serialize/DumpSerializedDataToText.cpp create mode 100644 Runtime/Serialize/DumpSerializedDataToText.h create mode 100644 Runtime/Serialize/FileCache.cpp create mode 100644 Runtime/Serialize/FileCache.h create mode 100644 Runtime/Serialize/FloatStringConversion.cpp create mode 100644 Runtime/Serialize/FloatStringConversion.h create mode 100644 Runtime/Serialize/IterateTypeTree.h create mode 100644 Runtime/Serialize/LoadProgress.h create mode 100644 Runtime/Serialize/PathNamePersistentManager.cpp create mode 100644 Runtime/Serialize/PathNamePersistentManager.h create mode 100644 Runtime/Serialize/PersistentManager.cpp create mode 100644 Runtime/Serialize/PersistentManager.h create mode 100644 Runtime/Serialize/Remapper.h create mode 100644 Runtime/Serialize/SerializationMetaFlags.h create mode 100644 Runtime/Serialize/SerializationTests.cpp create mode 100644 Runtime/Serialize/SerializeConversion.h create mode 100644 Runtime/Serialize/SerializeTraits.h create mode 100644 Runtime/Serialize/SerializeTraitsBase.h create mode 100644 Runtime/Serialize/SerializeUtility.h create mode 100644 Runtime/Serialize/SerializedFile.cpp create mode 100644 Runtime/Serialize/SerializedFile.h create mode 100644 Runtime/Serialize/SerializedFileTests.cpp create mode 100644 Runtime/Serialize/SwapEndianArray.h create mode 100644 Runtime/Serialize/SwapEndianBytes.h create mode 100644 Runtime/Serialize/TransferFunctionFwd.h create mode 100644 Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp create mode 100644 Runtime/Serialize/TransferFunctions/ProxyTransfer.h create mode 100644 Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp create mode 100644 Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp create mode 100644 Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h create mode 100644 Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp create mode 100644 Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp create mode 100644 Runtime/Serialize/TransferFunctions/SafeBinaryRead.h create mode 100644 Runtime/Serialize/TransferFunctions/SerializeTransfer.h create mode 100644 Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp create mode 100644 Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h create mode 100644 Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp create mode 100644 Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h create mode 100644 Runtime/Serialize/TransferFunctions/TransferBase.h create mode 100644 Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp create mode 100644 Runtime/Serialize/TransferFunctions/TransferNameConversions.h create mode 100644 Runtime/Serialize/TransferFunctions/YAMLRead.cpp create mode 100644 Runtime/Serialize/TransferFunctions/YAMLRead.h create mode 100644 Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp create mode 100644 Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h create mode 100644 Runtime/Serialize/TransferFunctions/YAMLWrite.cpp create mode 100644 Runtime/Serialize/TransferFunctions/YAMLWrite.h create mode 100644 Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp create mode 100644 Runtime/Serialize/TransferUtility.cpp create mode 100644 Runtime/Serialize/TransferUtility.h create mode 100644 Runtime/Serialize/TypeTree.cpp create mode 100644 Runtime/Serialize/TypeTree.h create mode 100644 Runtime/Serialize/WriteData.h create mode 100644 Runtime/Serialize/WriteTypeToBuffer.h (limited to 'Runtime/Serialize') diff --git a/Runtime/Serialize/AwakeFromLoadQueue.cpp b/Runtime/Serialize/AwakeFromLoadQueue.cpp new file mode 100644 index 0000000..fdb31de --- /dev/null +++ b/Runtime/Serialize/AwakeFromLoadQueue.cpp @@ -0,0 +1,395 @@ +#include "UnityPrefix.h" +#include "AwakeFromLoadQueue.h" +#include "Runtime/Profiler/Profiler.h" +#include "Runtime/Mono/MonoBehaviour.h" +#include "Runtime/Mono/MonoScript.h" +#include "Runtime/Misc/BuildSettings.h" +#if UNITY_EDITOR +#include "Editor/Src/Prefabs/PrefabBackwardsCompatibility.h" +#endif + +PROFILER_INFORMATION(gAwakeFromLoadQueue, "Loading.AwakeFromLoad", kProfilerLoading) + +AwakeFromLoadQueue::AwakeFromLoadQueue (MemLabelRef label) +{ + for(int i=0;i(target.GetInstanceID())) + return true; + } + + return false; +} + + +void AwakeFromLoadQueue::Add (Object& target, TypeTree* oldType, bool safeBinaryLoaded, AwakeFromLoadMode awakeOverride) +{ + Item item; + item.registerObjectPtr = ⌖ + item.objectPPtr = ⌖ + item.classID = target.GetClassID(); + #if UNITY_EDITOR + item.oldType = oldType; + item.safeBinaryLoaded = safeBinaryLoaded; + item.awakeModeOverride = awakeOverride; + #else + Assert(awakeOverride == -1); + #endif + + int queueIndex = DetermineQueueIndex(item.classID); + + m_ItemArrays[queueIndex].push_back(item); +} + +bool SortItemByInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return lhs.objectPPtr.GetInstanceID() < rhs.objectPPtr.GetInstanceID(); +} + +static int GetScriptExecutionOrder (int instanceID) +{ + #if ENABLE_SCRIPTING + MonoBehaviour* behaviour = dynamic_instanceID_cast (instanceID); + if (behaviour != NULL) + { + MonoScript* script = behaviour->GetScript(); + if (script) + return script->GetExecutionOrder(); + } + #endif + return 0; +} + +bool AwakeFromLoadQueue::SortBehaviourByExecutionOrderAndInstanceID (int lhs, int rhs) +{ + int lhsExecutionOrder = GetScriptExecutionOrder(lhs); + int rhsExecutionOrder = GetScriptExecutionOrder(rhs); + + if (lhsExecutionOrder != rhsExecutionOrder) + return lhsExecutionOrder < rhsExecutionOrder; + else + return lhs < rhs; +} + +bool SortBehaviourByExecutionOrderAndReverseInstanceID (int lhs, int rhs) +{ + int lhsExecutionOrder = GetScriptExecutionOrder(lhs); + int rhsExecutionOrder = GetScriptExecutionOrder(rhs); + + if (lhsExecutionOrder != rhsExecutionOrder) + return lhsExecutionOrder < rhsExecutionOrder; + else + return lhs > rhs; +} + +static bool SortBehaviourItemByExecutionOrderAndInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return AwakeFromLoadQueue::SortBehaviourByExecutionOrderAndInstanceID(lhs.objectPPtr.GetInstanceID(), rhs.objectPPtr.GetInstanceID()); +} + +static bool SortBehaviourItemByExecutionOrderAndReverseInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return SortBehaviourByExecutionOrderAndReverseInstanceID(lhs.objectPPtr.GetInstanceID(), rhs.objectPPtr.GetInstanceID()); +} + +///@TODO: Should check consistency always be called immediately before calling Awake??? + +void AwakeFromLoadQueue::PersistentManagerAwakeFromLoad (AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + for (int i=0;i= oldDstSize || dst[d].objectPPtr.GetInstanceID() != src[i].objectPPtr.GetInstanceID()) + { + dst.push_back(src[i]); + } + // else -> The object is in the destination array already. + } +} + +void AwakeFromLoadQueue::InsertAwakeFromLoadQueue (AwakeFromLoadQueue& awakeFromLoadQueue, AwakeFromLoadMode awakeOverride) +{ + for (int i=0;i (ptr); + if (prefab) + prefab->PatchPrefabBackwardsCompatibility(); + + EditorExtension* extension = dynamic_pptr_cast (ptr); + if (extension) + extension->PatchPrefabBackwardsCompatibility(); + } + } + } +} +#endif + + +void AwakeFromLoadQueue::InvokePersistentManagerAwake (Item* objects, unsigned size, AwakeFromLoadMode awakeMode, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + #if DEBUGMODE + int previousInstanceID = 0; + #endif + + for (int i=0;iCheckConsistency (); + + #endif + + AwakeFromLoadMode objectAwakeMode = awakeMode; + #if UNITY_EDITOR + if (objects[i].awakeModeOverride != kDefaultAwakeFromLoadInvalid) + objectAwakeMode = objects[i].awakeModeOverride; + #endif + + ptr->AwakeFromLoad (objectAwakeMode); + ptr->ClearPersistentDirty (); + } +} + +void AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (Object& o, TypeTree* oldType, AwakeFromLoadMode awakeMode, bool safeBinaryLoaded, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + PROFILER_AUTO(gAwakeFromLoadQueue, &o) + + #if UNITY_EDITOR + // Loaded with SafeBinaryRead thus needs the callback + if (safeBinaryLoaded && safeBinaryCallback != NULL) + safeBinaryCallback (o, *oldType); + o.CheckConsistency (); + #endif + + o.AwakeFromLoad (awakeMode); + o.ClearPersistentDirty (); +} + +void AwakeFromLoadQueue::InvokeAwakeFromLoad (Item* objects, unsigned size, AwakeFromLoadMode mode) +{ + for (int i=0;iAwakeFromLoad(objects[i].awakeModeOverride); + continue; + } + + #endif + + + ptr->AwakeFromLoad(mode); + } + } +} + +void AwakeFromLoadQueue::InvokeCheckConsistency (Item* objects, unsigned size) +{ + for (int i=0;iCheckConsistency(); + } +} + +void AwakeFromLoadQueue::ExtractAllObjects (dynamic_array >& outObjects) +{ + Assert(outObjects.empty()); + + int count = 0; + for (int q=0;q objectPPtr; + ClassIDType classID; + +#if UNITY_EDITOR + TypeTree* oldType; + bool safeBinaryLoaded; + AwakeFromLoadMode awakeModeOverride; +#endif + }; + + AwakeFromLoadQueue (MemLabelRef label); + + typedef dynamic_array ItemArray; + + typedef void SafeBinaryReadCallbackFunction (Object& object, const TypeTree& oldTypeTree); + + bool IsInQueue(Object& target); + + void Reserve (unsigned size); + void Add (Object& target, TypeTree* oldType = NULL, bool safeBinaryLoaded = false, AwakeFromLoadMode awakeOverride = kDefaultAwakeFromLoadInvalid); + + static void PersistentManagerAwakeSingleObject (Object& o, TypeTree* oldType, AwakeFromLoadMode awakeMode, bool safeBinaryLoaded, SafeBinaryReadCallbackFunction* safeBinaryCallback); + + void PersistentManagerAwakeFromLoad (AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + void PersistentManagerAwakeFromLoad (int queueIndex, AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + + void ClearQueue (int queueIndex); + void Clear (); + + void AwakeFromLoad (AwakeFromLoadMode mode); + void CheckConsistency (); + + void RegisterObjectInstanceIDs (); + + #if UNITY_EDITOR + void PatchPrefabBackwardsCompatibility (); + void InsertAwakeFromLoadQueue (AwakeFromLoadQueue& awakeFromLoadQueue, AwakeFromLoadMode awakeOverride); + #endif + + void ExtractAllObjects (dynamic_array >& outObjects); + + ItemArray& GetItemArray (int queueIndex) { return m_ItemArrays[queueIndex]; } + + static bool SortBehaviourByExecutionOrderAndInstanceID (int lhs, int rhs); + +private: + int DetermineQueueIndex(ClassIDType classID); + + static void InvokeAwakeFromLoad (Item* objects, unsigned size, AwakeFromLoadMode mode); + static void InvokeCheckConsistency (Item* objects, unsigned size); + static void InvokePersistentManagerAwake (Item* objects, unsigned size, AwakeFromLoadMode awakeMode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + static void RegisterObjectInstanceIDsInternal (Item* objects, unsigned size); + + void InsertAwakeFromLoadQueue (dynamic_array& src, dynamic_array& dst, AwakeFromLoadMode awakeOverride); + + + ItemArray m_ItemArrays[kMaxQueues]; +}; + +#endif \ No newline at end of file diff --git a/Runtime/Serialize/Blobification/BlobSize.h b/Runtime/Serialize/Blobification/BlobSize.h new file mode 100644 index 0000000..a7aaf6d --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobSize.h @@ -0,0 +1,161 @@ +#ifndef BLOBSIZE_H +#define BLOBSIZE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" + +struct ReduceCopyData; + +class BlobSize : public TransferBase +{ +private: + + typedef OffsetPtr::offset_type offset_type; + + size_t m_Size; + bool m_IgnorePtr; + bool m_HasDebugOffsetPtr; + bool m_Use64Ptr; + int m_TargetPlatform; + + std::size_t AlignAddress(std::size_t addr, std::size_t align) + { + return addr + ((~addr + 1U) & (align - 1U)); + } + +public: + + + BlobSize (bool hasDebugOffsetPtr, bool use64BitOffsetPtr) + : m_Size (0) + , m_IgnorePtr (false) + , m_HasDebugOffsetPtr (hasDebugOffsetPtr) + , m_Use64Ptr (use64BitOffsetPtr) { } + + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + int GetFlags () { return m_Flags; } + bool IsWritingGameReleaseData () + { + return true; + } + bool IsSerializingForGameRelease () + { + return true; + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + if (platform == kBuildAnyPlayerData) + return m_TargetPlatform >= kBuildValidPlayer; + else + return m_TargetPlatform == platform; + } + + BuildTargetPlatform GetBuildingTargetPlatform () { return static_cast(m_TargetPlatform); } + + template + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + template + void TransferBasicData (T& data); + + template + void TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData); + + template + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template + static size_t CalculateSize (T& data, bool hasDebugOffsetPtr, bool use64BitOffsetPtr) + { + BlobSize sizer (hasDebugOffsetPtr, use64BitOffsetPtr); + sizer.Transfer(data, "Base"); + + // size_t size = sizeof(T); + // Assert( sizeof(T) == sizer.m_Size); + + return sizer.m_Size; + } + + template friend class BlobSizeTransferSTLStyleArrayImpl; +}; + + +template class BlobSizeTransferSTLStyleArrayImpl +{ +public: + void operator()(T& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + AssertString ("STL array are not support for BlobWrite"); + } +}; + +template class BlobSizeTransferSTLStyleArrayImpl< OffsetPtrArrayTransfer > +{ +public: + void operator()(OffsetPtrArrayTransfer& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + transfer.m_IgnorePtr = false; + } +}; + +template class BlobSizeTransferSTLStyleArrayImpl< StaticArrayTransfer > +{ +public: + void operator()(StaticArrayTransfer& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + transfer.m_Size = transfer.AlignAddress(transfer.m_Size, ALIGN_OF(T)); + transfer.m_Size += sizeof(T)*data.size(); + } +}; + + +template inline +void BlobSize::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlags) +{ + BlobSizeTransferSTLStyleArrayImpl transfer; + transfer(data, metaFlags, *this); +} + +template inline +void BlobSize::Transfer (T& data, const char* name, TransferMetaFlags) +{ + if (m_IgnorePtr) + { + m_IgnorePtr = false; + return; + } + + m_Size = AlignAddress(m_Size, SerializeTraits::GetAlignOf() ); + + SerializeTraits::Transfer (data, *this); + + m_Size = AlignAddress(m_Size, SerializeTraits::GetAlignOf() ); +} + +template inline +void BlobSize::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits::Transfer (data, *this); +} + +template inline +void BlobSize::TransferBasicData (T& srcData) +{ + m_Size += sizeof(T); +} + +template inline +void BlobSize::TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData) +{ + m_Size += m_Use64Ptr ? sizeof(SInt64) : sizeof(SInt32); + + if (m_HasDebugOffsetPtr) + m_Size += sizeof(void*); + + if (isValidPtr) + m_IgnorePtr = true; +} + +#endif diff --git a/Runtime/Serialize/Blobification/BlobTests.cpp b/Runtime/Serialize/Blobification/BlobTests.cpp new file mode 100644 index 0000000..8b0a2d3 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobTests.cpp @@ -0,0 +1,358 @@ +#include "UnityPrefix.h" + +#if ENABLE_UNIT_TESTS && !(UNITY_LINUX && UNITY_64) +#include "External/UnitTest++/src/UnitTest++.h" +#include +#include "Runtime/Math/Vector3.h" +#include "Runtime/Math/Simd/math.h" +#include "Runtime/mecanim/defs.h" +#include "Runtime/mecanim/types.h" +#include "Runtime/Allocator/MemoryManager.h" +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/Blobification/offsetptr.h" +#include "Runtime/Serialize/Blobification/BlobWrite.h" + +template TYPE* Construct() +{ + #if ENABLE_MEMORY_MANAGER + void* _where = GetMemoryManager().Allocate (sizeof(TYPE), ALIGN_OF(TYPE), MemLabelId(kMemNewDeleteId, NULL), kAllocateOptionNone, "Construct"); + #else + void* _where = malloc(sizeof(TYPE)); + #endif + return new(_where) TYPE; +} + +template TYPE* ConstructArray(size_t num) +{ + #if ENABLE_MEMORY_MANAGER + void* _where = GetMemoryManager().Allocate (sizeof(TYPE)*num, ALIGN_OF(TYPE), MemLabelId(kMemNewDeleteId, NULL), kAllocateOptionNone, "ConstructArray"); + #else + void* _where = malloc(sizeof(TYPE)*num); + #endif + return new(_where) TYPE[num]; +} + +template void Destruct(TYPE* p) +{ + #if ENABLE_MEMORY_MANAGER + GetMemoryManager().Deallocate (p, MemLabelId(kMemNewDeleteId, NULL)); + #else + free(p); + #endif +} + + +struct SampleDataA +{ + enum { + kLastIndex = 20 + }; + + int intValue1; + math::float4 float4Value; // (intentially unaligned) + Vector3f vector3; + + mecanim::uint32_t index[kLastIndex]; + + OffsetPtr nullPtr; + + OffsetPtr floatPtr; + + mecanim::uint32_t arraySize; + OffsetPtr array; + + mecanim::uint32_t emptyArraySize; + OffsetPtr emptyArray; + + int intValue2; + + DECLARE_SERIALIZE(SampleData) +}; + +struct SampleData +{ + int intValue1; // 0 + math::float4 float4Value; // 16 (intentially unaligned) + Vector3f vector3; // 32 + + OffsetPtr nullPtr; // 44 + + OffsetPtr floatPtr; // 52 + + mecanim::uint32_t arraySize; // 60 + OffsetPtr array; // 64 + + mecanim::uint32_t emptyArraySize; // 72 + OffsetPtr emptyArray; // 76 + + mecanim::uint32_t sampleDataASize; // 84 + OffsetPtr sampleDataA; // 88 + + mecanim::uint32_t sampleDataAHandleSize; // 96 + OffsetPtr > sampleDataAHandle; // 100 + + int intValue2; // 108 + + DECLARE_SERIALIZE(SampleData) +}; + + + +template inline +void SampleDataA::Transfer(TransferFunction& transfer) +{ + TRANSFER(intValue1); + TRANSFER(float4Value); + TRANSFER(vector3); + + STATIC_ARRAY_TRANSFER(mecanim::uint32_t, index, kLastIndex); + + TRANSFER(nullPtr); + TRANSFER(floatPtr); + + TRANSFER_BLOB_ONLY(arraySize); + MANUAL_ARRAY_TRANSFER2(float, array, arraySize); + + + TRANSFER_BLOB_ONLY(emptyArraySize); + MANUAL_ARRAY_TRANSFER2(math::float4, emptyArray, emptyArraySize); + + TRANSFER(intValue2); +} + + +template inline +void SampleData::Transfer(TransferFunction& transfer) +{ + TRANSFER(intValue1); + TRANSFER(float4Value); + TRANSFER(vector3); + + TRANSFER(nullPtr); + TRANSFER(floatPtr); + + TRANSFER_BLOB_ONLY(arraySize); + MANUAL_ARRAY_TRANSFER2(double, array, arraySize); + + + TRANSFER_BLOB_ONLY(emptyArraySize); + MANUAL_ARRAY_TRANSFER2(math::float4, emptyArray, emptyArraySize); + + TRANSFER_BLOB_ONLY(sampleDataASize); + MANUAL_ARRAY_TRANSFER2(SampleDataA, sampleDataA, sampleDataASize); + + TRANSFER_BLOB_ONLY(sampleDataAHandleSize); + MANUAL_ARRAY_TRANSFER2(OffsetPtr, sampleDataAHandle, sampleDataAHandleSize); + + TRANSFER(intValue2); +} + +static void SetupTestDataA (SampleDataA& sourceData) +{ + sourceData.intValue1 = 1; + sourceData.float4Value = math::float4(1, 2, 3, 4); + sourceData.vector3 = Vector3f(1,2,3); + + mecanim::uint32_t i; + for(i=0; i(sourceData.sampleDataASize); + for(int i=0;i [2]; + sourceData.sampleDataAHandle[0] = Construct(); + SetupTestDataA(*sourceData.sampleDataAHandle[0]); + sourceData.sampleDataAHandle[1] = Construct(); + SetupTestDataA(*sourceData.sampleDataAHandle[1]); +} + + +static void DeleteTestData (SampleData& sourceData) +{ + for(int i=0;i(&deserialized) % ALIGN_OF(SampleDataA) == 0 ); + CHECK (deserialized.intValue1 == 1); + CHECK (dot(deserialized.float4Value - math::float4(1, 2, 3, 4)) == math::float1::zero()); + + mecanim::uint32_t i; + for(i=0; i(&deserialized) % ALIGN_OF(SampleData) == 0); + CHECK (deserialized.intValue1 == 1); + CHECK (dot(deserialized.float4Value - math::float4(1, 2, 3, 4)) == math::float1::zero() ); + + CHECK (deserialized.nullPtr.IsNull()); + + float ptr = *deserialized.floatPtr; + CHECK (ptr == 5.5F); + + double* array = deserialized.array.Get(); + CHECK (array[0] == 6.5); + CHECK (array[1] == 7.5); + CHECK (array[2] == 8.5); + + CHECK (deserialized.emptyArray.IsNull()); + CHECK (deserialized.emptyArraySize == 0); + + CHECK (deserialized.vector3 == Vector3f(1,2,3)); + CHECK (deserialized.intValue2 == 2); + + CHECK (deserialized.sampleDataASize == 4); + for(int i=0;i (data.begin())); + + // Generate blob with reduce copy + BlobWrite::container_type dataReduced; + BlobWrite blobWriteReduce (dataReduced, kNoTransferInstructionFlags, kBuildNoTargetPlatform); + blobWriteReduce.SetReduceCopy(true); + blobWriteReduce.Transfer(sourceData, "Base"); + TestData(*reinterpret_cast (dataReduced.begin())); + + // Ensure reduced blob is actually smaller. + CHECK (dataReduced.size() < data.size()); + + // Ensure that 64 bit data is larger than non-64 bit data + BlobWrite::container_type data64; + BlobWrite blobWrite64 (data64, kNoTransferInstructionFlags, kBuildStandaloneWin64Player); + blobWrite64.Transfer(sourceData, "Base"); + + BlobWrite::container_type data32; + BlobWrite blobWrite32 (data32, kNoTransferInstructionFlags, kBuildStandaloneWinPlayer); + blobWrite32.Transfer(sourceData, "Base"); + CHECK (data64.size() > data32.size()); + + DeleteTestData (sourceData); + } + + TEST (Blobification_OffsetPtr) + { + OffsetPtr* ptrHigh = new OffsetPtr; + OffsetPtr* ptrLow = new OffsetPtr; + + size_t* ptrH = reinterpret_cast(std::numeric_limits::max()-4); + size_t* ptrL = reinterpret_cast(4); + + ptrHigh->reset(ptrH); + ptrLow->reset(ptrL); + + size_t h = reinterpret_cast(ptrHigh->Get()); + size_t l = reinterpret_cast(ptrLow->Get()); + + + CHECK (h == std::numeric_limits::max()-4); + CHECK (l == 4); + + delete ptrHigh; + delete ptrLow; + } +} + +#endif //ENABLE_UNIT_TESTS diff --git a/Runtime/Serialize/Blobification/BlobWrite.cpp b/Runtime/Serialize/Blobification/BlobWrite.cpp new file mode 100644 index 0000000..dcb7705 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWrite.cpp @@ -0,0 +1,148 @@ +#include "UnityPrefix.h" +#include "BlobWrite.h" +#include "Configuration/UnityConfigure.h" +#include "BlobWriteTargetSupport.h" + +BlobWrite::BlobWrite (container_type& blob, TransferInstructionFlags flags, BuildTargetPlatform targetPlatform) +: m_Blob(blob), + m_CopyData(true), + m_ReduceCopy(false), + m_TargetPlatform(targetPlatform) +{ + m_Flags = false; + m_SwapEndianess = m_Flags & kSwapEndianess; + m_Use64BitOffsetPtr = IsBuildTarget64BitBlob (targetPlatform); +} + +void BlobWrite::Push (size_t size, void* srcDataPtr, size_t align) +{ + Assert (m_CopyData); + + size_t offset = AlignAddress(m_Blob.size(), align); + m_Context.push( TypeContext(offset, 0, reinterpret_cast (srcDataPtr), size) ); + m_Blob.resize_initialized(offset + size, 0); + m_CopyData = false; +} + +void BlobWrite::WritePtrValueAtLocation (size_t locationInBlob, SInt64 value) +{ + if (m_Use64BitOffsetPtr) + { + SInt64 offset64 = value; + + if (m_SwapEndianess) + SwapEndianBytes(offset64); + memcpy (&m_Blob[locationInBlob], &offset64, sizeof(offset64)); + } + else + { + SInt32 offset32 = value; + + if (m_SwapEndianess) + SwapEndianBytes(offset32); + memcpy (&m_Blob[locationInBlob], &offset32, sizeof(offset32)); + } +} + + +void BlobWrite::TransferPtrImpl (bool isValidPtr, ReduceCopyData* reduceCopyData, size_t alignOfT) +{ + Assert(!m_CopyData); + // When the data is null we will not call Transfer. + m_CopyData = isValidPtr; + + // Need to update OffsetPtr's member 'mOffset' + // compute member offset in memory buffer + size_t dataPosition = AlignAddress(m_Blob.size(), alignOfT); + size_t offset = GetActiveOffset(); + offset = dataPosition - offset; + if (!isValidPtr) + offset = 0; + + // Write the ptr + WritePtrValueAtLocation(GetActiveOffset (), offset); + + // Setup reduce copy data for later use by ReduceCopyImpl + if (reduceCopyData != NULL) + { + if (isValidPtr) + { + reduceCopyData->ptrPosition = GetActiveOffset(); + reduceCopyData->dataStart = dataPosition; + reduceCopyData->blobSize = m_Blob.size(); + } + else + { + reduceCopyData->ptrPosition = 0xFFFFF; + reduceCopyData->dataStart = 0xFFFFF; + reduceCopyData->blobSize = 0xFFFFF; + } + } + + + // Offset write location in the blob + m_Context.top().m_Offset += m_Use64BitOffsetPtr ? sizeof(SInt64) : sizeof(SInt32); + if (HasOffsetPtrWithDebugPtr()) + m_Context.top().m_Offset += sizeof(void*); +} + +bool BlobWrite::HasOffsetPtrWithDebugPtr () const +{ + return m_TargetPlatform == kBuildNoTargetPlatform; +} + +bool BlobWrite::AllowDataLayoutValidation () const +{ + size_t targetOffsetPtrSize = Use64BitOffsetPtr () ? sizeof(SInt64) : sizeof(SInt32); + if (HasOffsetPtrWithDebugPtr ()) + targetOffsetPtrSize += sizeof(void*); + + size_t srcOffsetPtrSize = sizeof(OffsetPtr); + + return targetOffsetPtrSize == srcOffsetPtrSize; +} + +// Ensure that the user has matching Transfer calls & Data layout in the struct +void BlobWrite::ValidateSerializedLayout (void* srcData, const char* name) +{ + UInt8* srcDataPtr = reinterpret_cast (srcData); + + // (float4 and some others transfer functions, transfer temporary data, we ignore layout checks on those and hope for the best) + int srcDataOffset = srcDataPtr - m_Context.top().m_SourceDataPtr; + if (srcDataOffset < 0 || srcDataOffset >= m_Context.top().m_SourceDataSize) + return; + + // When targeting a platform with a different layout than our own, obviously these checks dont make sense... + if (!AllowDataLayoutValidation ()) + return; + + int blobOffset = m_Context.top().m_Offset; + if (srcDataOffset != blobOffset) + { + AssertString(Format("BlobWrite: Transfer '%s' is not called in the same order as the struct is laid out. Expected: %d got: %d ", name, srcDataOffset, blobOffset)); + } +} + +void BlobWrite::ReduceCopyImpl (const ReduceCopyData& reduce, size_t alignOfT) +{ + if (!m_ReduceCopy || reduce.dataStart == 0xFFFFF) + return; + + // Find any data in the blob that matches the last written data. + // if we find it, delete it again and reference the previous memory block instead + size_t dataSize = m_Blob.size() - reduce.dataStart; + for (int i=0;i < reduce.dataStart;i+=alignOfT) + { + if (memcmp(&m_Blob[i], &m_Blob[reduce.dataStart], dataSize) == 0) + { + // Update offset pointer + SInt64 offset = i - reduce.ptrPosition; + WritePtrValueAtLocation (reduce.ptrPosition, offset); + + // resize blob based on the reduce copy + m_Blob.resize_initialized(reduce.blobSize, 0); + + return; + } + } +} diff --git a/Runtime/Serialize/Blobification/BlobWrite.h b/Runtime/Serialize/Blobification/BlobWrite.h new file mode 100644 index 0000000..56a0340 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWrite.h @@ -0,0 +1,220 @@ +#ifndef BLOBWRITE_H +#define BLOBWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/SwapEndianBytes.h" +#include "Runtime/Animation/MecanimArraySerialization.h" +#include "offsetptr.h" +#include "BlobSize.h" +#include "ReduceCopyData.h" +#include +#include "Runtime/Modules/ExportModules.h" + +class EXPORT_COREMODULE BlobWrite : public TransferBase +{ +public: + + typedef dynamic_array container_type; + +private: + + container_type& m_Blob; + int m_TargetPlatform; + bool m_CopyData; + bool m_ReduceCopy; + bool m_Use64BitOffsetPtr; + bool m_SwapEndianess; + + struct TypeContext + { + TypeContext(size_t root, size_t offset, UInt8* srcDataPtr,size_t srcDataSize):m_Root(root),m_Offset(offset),m_SourceDataPtr(srcDataPtr),m_SourceDataSize(srcDataSize) {} + + size_t m_Root; + size_t m_Offset; + + UInt8* m_SourceDataPtr; + size_t m_SourceDataSize; + }; + std::stack m_Context; + + std::size_t AlignAddress(std::size_t addr, std::size_t align) + { + return addr + ((~addr + 1U) & (align - 1U)); + } + + size_t GetActiveOffset () const + { + return m_Context.top().m_Root + m_Context.top().m_Offset; + } + + UInt8* GetActiveBlobPtr () + { + return &m_Blob[GetActiveOffset ()]; + } + + void WritePtrValueAtLocation (size_t locationInBlob, SInt64 value); + + void ValidateSerializedLayout (void* srcData, const char* name); + + void Push (size_t size, void* srcDataPtr, size_t align); + + void TransferPtrImpl (bool isValidPtr, ReduceCopyData* reduceCopyData, size_t alignOfT); + void ReduceCopyImpl (const ReduceCopyData& reduce, size_t alignOfT); + + bool HasOffsetPtrWithDebugPtr () const; + bool Use64BitOffsetPtr() const { return m_Use64BitOffsetPtr; } + bool AllowDataLayoutValidation () const; + +public: + + BlobWrite (container_type& blob, TransferInstructionFlags flags, BuildTargetPlatform targetPlatform); + + void SetReduceCopy (bool reduce) { m_ReduceCopy = reduce; } + + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return m_SwapEndianess; } + bool IsWritingGameReleaseData () + { + return true; + } + bool IsSerializingForGameRelease () + { + return true; + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + if (platform == kBuildAnyPlayerData) + return m_TargetPlatform >= kBuildValidPlayer; + else + return m_TargetPlatform == platform; + } + + BuildTargetPlatform GetBuildingTargetPlatform () { return static_cast(m_TargetPlatform); } + + template + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + template + void TransferBasicData (T& data); + + template + void TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData); + + template + void ReduceCopy (const ReduceCopyData& reduce); + + template + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template friend class BlobWriteTransferSTLStyleArrayImpl; +}; + +template class BlobWriteTransferSTLStyleArrayImpl +{ +public: + void operator()(T& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + AssertString ("STL array are not support for BlobWrite"); + } +}; + +template class BlobWriteTransferSTLStyleArrayImpl< OffsetPtrArrayTransfer > +{ +public: + void operator()(OffsetPtrArrayTransfer& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + if (data.size() == 0) + { + Assert (!transfer.m_CopyData); + return; + } + Assert (transfer.m_CopyData); + + size_t arrayByteSize = BlobSize::CalculateSize(*data.begin(), transfer.HasOffsetPtrWithDebugPtr(), transfer.Use64BitOffsetPtr()) * data.size(); + transfer.Push(arrayByteSize, &*data.begin(), ALIGN_OF( typename OffsetPtrArrayTransfer::value_type )); + + typename OffsetPtrArrayTransfer::iterator end = data.end (); + for (typename OffsetPtrArrayTransfer::iterator i = data.begin ();i != end;++i) + transfer.Transfer (*i, "data"); + + transfer.m_Context.pop(); + } +}; + +template class BlobWriteTransferSTLStyleArrayImpl< StaticArrayTransfer > +{ +public: + void operator()(StaticArrayTransfer& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + typename StaticArrayTransfer::iterator end = data.end (); + for (typename StaticArrayTransfer::iterator i = data.begin ();i != end;++i) + transfer.Transfer (*i, "data"); + } +}; + +template inline +void BlobWrite::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlags) +{ + BlobWriteTransferSTLStyleArrayImpl transfer; + transfer(data, metaFlags, *this); +} + +template inline +void BlobWrite::Transfer (T& data, const char* name, TransferMetaFlags) +{ + bool copyData = m_CopyData; + if (m_CopyData) + Push(BlobSize::CalculateSize(data, HasOffsetPtrWithDebugPtr(), Use64BitOffsetPtr()), &data, SerializeTraits::GetAlignOf() ); + + // Follow natural c++ alignment + size_t head = m_Context.top().m_Root; + size_t& offset = m_Context.top().m_Offset; + // always align head + offset not only offset otherwise you may get wrong align for nested data structure + offset = AlignAddress(head + offset, SerializeTraits::GetAlignOf()) - head; + + ValidateSerializedLayout(&data, name); + + SerializeTraits::Transfer (data, *this); + + if (copyData) + m_Context.pop(); +} + +template inline +void BlobWrite::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits::Transfer (data, *this); +} + +template inline +void BlobWrite::TransferBasicData (T& srcData) +{ + Assert(m_Blob.size() >= GetActiveOffset() + sizeof(T)); + + // Write basic data into blob & endianswap + UInt8* blobPtr = GetActiveBlobPtr(); + memcpy(blobPtr, &srcData, sizeof(T)); + if (m_SwapEndianess) + SwapEndianBytes(*reinterpret_cast (blobPtr)); + + m_Context.top().m_Offset += sizeof(T); +} + +template inline +void BlobWrite::TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData) +{ + TransferPtrImpl (isValidPtr, reduceCopyData, ALIGN_OF(T)); +} + +template inline +void BlobWrite::ReduceCopy (const ReduceCopyData& reduce) +{ + ReduceCopyImpl(reduce, ALIGN_OF(T)); +} + +#endif diff --git a/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp new file mode 100644 index 0000000..b66b941 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" +#include "BlobWriteTargetSupport.h" + +bool DoesBuildTargetSupportBlobification (BuildTargetPlatform target, TransferInstructionFlags flags) +{ + // If we are writing typetrees, then we can't use blobification + bool writeTypeTree = (flags & kDisableWriteTypeTree) == 0; + if (writeTypeTree) + return false; + + + // Webplayer & Editor should never use blobification + Assert(target != kBuildWebPlayerLZMA && target != kBuildWebPlayerLZMAStreamed && target != kBuildAnyPlayerData || target == kBuildNoTargetPlatform); + return true; +} + +bool IsBuildTarget64BitBlob (BuildTargetPlatform target) +{ + Assert(target != kBuildAnyPlayerData && target != kBuildWebPlayerLZMA && target != kBuildWebPlayerLZMAStreamed); + + // Building blob for the editor (Choose whatever we are running with) + if (target == kBuildNoTargetPlatform) + return sizeof(size_t) == sizeof(UInt64); + + // Known 64 bit platform? + bool target64Bit = target == kBuildMetroPlayerX64 || target == kBuildStandaloneWin64Player || target == kBuildStandaloneLinux64 || target == kBuildStandaloneLinuxUniversal; + if (target64Bit) + return true; + + return false; +} \ No newline at end of file diff --git a/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h new file mode 100644 index 0000000..76191f5 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h @@ -0,0 +1,6 @@ +#pragma once + +#include "Runtime/Serialize/SerializationMetaFlags.h" + +bool DoesBuildTargetSupportBlobification (BuildTargetPlatform target, TransferInstructionFlags flags); +bool IsBuildTarget64BitBlob (BuildTargetPlatform target); \ No newline at end of file diff --git a/Runtime/Serialize/Blobification/OffsetPtrTest.cpp b/Runtime/Serialize/Blobification/OffsetPtrTest.cpp new file mode 100644 index 0000000..9ddd2a8 --- /dev/null +++ b/Runtime/Serialize/Blobification/OffsetPtrTest.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" + +#include "Runtime/Serialize/SerializeTraits.h" +#include "offsetptr.h" + + +#include + + +void TestOffsetPtr () +{ + OffsetPtr* ptrHigh = new OffsetPtr; + OffsetPtr* ptrLow = new OffsetPtr; + + size_t* ptrH = reinterpret_cast(std::numeric_limits::max()-4); + size_t* ptrL = reinterpret_cast(4); + + ptrHigh->reset(ptrH); + ptrLow->reset(ptrL); + + size_t h = reinterpret_cast(ptrHigh->Get()); + size_t l = reinterpret_cast(ptrLow->Get()); + + + Assert(h == std::numeric_limits::max()-4); + Assert(l == 4); + + delete ptrHigh; + delete ptrLow; + +} diff --git a/Runtime/Serialize/Blobification/ReduceCopyData.h b/Runtime/Serialize/Blobification/ReduceCopyData.h new file mode 100644 index 0000000..ae4952f --- /dev/null +++ b/Runtime/Serialize/Blobification/ReduceCopyData.h @@ -0,0 +1,8 @@ +#pragma once + +struct ReduceCopyData +{ + size_t ptrPosition; + size_t dataStart; + size_t blobSize; +}; diff --git a/Runtime/Serialize/Blobification/offsetptr.h b/Runtime/Serialize/Blobification/offsetptr.h new file mode 100644 index 0000000..7ef1809 --- /dev/null +++ b/Runtime/Serialize/Blobification/offsetptr.h @@ -0,0 +1,257 @@ +#pragma once + +#include "Runtime/mecanim/memory.h" +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/SerializeUtility.h" +#include "Runtime/Serialize/SerializeTraitsBase.h" +#include "Runtime/Serialize/TransferFunctionFwd.h" +#include "Runtime/Utilities/TypeUtilities.h" +#include "ReduceCopyData.h" + +template +class OffsetPtr +{ +public: + typedef TYPE value_type; + typedef TYPE* ptr_type; + typedef TYPE const* const_ptr_type; + typedef TYPE& reference_type; + typedef TYPE const& const_reference_type; + typedef size_t offset_type; + + OffsetPtr():m_Offset(0),m_DebugPtr(0) + { + } + + OffsetPtr (const OffsetPtr& ptr):m_Offset(ptr.m_Offset) + { + } + + OffsetPtr& operator = (const OffsetPtr& ptr) + { + m_Offset = ptr.m_Offset; + return *this; + } + + void reset(ptr_type ptr) + { + m_Offset = ptr != 0 ? reinterpret_cast(ptr) - reinterpret_cast(this) : 0; +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + } + + OffsetPtr& operator = (const ptr_type ptr) + { + reset (ptr); + return *this; + } + + ptr_type operator->() + { + ptr_type ptr = reinterpret_cast(reinterpret_cast(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return ptr; + } + const_ptr_type operator->()const + { + return reinterpret_cast(reinterpret_cast(this) + m_Offset); + } + + reference_type operator*() + { + ptr_type ptr = reinterpret_cast(reinterpret_cast(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return *ptr; + } + + const_reference_type operator*()const + { + return *reinterpret_cast(reinterpret_cast(this) + m_Offset); + } + + value_type& operator[](std::size_t i ) + { + assert(i != std::numeric_limits::max()); + ptr_type ptr = reinterpret_cast(reinterpret_cast(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return ptr[i]; + } + + value_type const& operator[](std::size_t i ) const + { + assert(i != std::numeric_limits::max()); + const_ptr_type ptr = reinterpret_cast(reinterpret_cast(this) + m_Offset); + return ptr[i]; + } + + bool IsNull()const + { + return m_Offset == 0; + } + + ptr_type Get() + { +#ifdef UNITY_EDITOR + m_DebugPtr = reinterpret_cast(reinterpret_cast(this) + m_Offset); +#endif + // TODO: serialize trait for offset ptr call begin and end which call OffsetPtr::Get + //Assert(!IsNull()); + return reinterpret_cast(reinterpret_cast(this) + m_Offset); + } + + const_ptr_type Get()const + { +#ifdef UNITY_EDITOR + m_DebugPtr = reinterpret_cast(reinterpret_cast(this) + m_Offset); +#endif + // TODO: serialize trait for offset ptr call begin and end which call OffsetPtr::Get + //Assert(!IsNull()); + return reinterpret_cast(reinterpret_cast(this) + m_Offset); + } + + + size_t get_size () const + { + return sizeof(TYPE); + } + +protected: + offset_type m_Offset; +#ifdef UNITY_EDITOR + mutable ptr_type m_DebugPtr; +#endif +}; + + +template +class SerializeTraits< OffsetPtr > : public SerializeTraitsBase< OffsetPtr > +{ + public: + + typedef OffsetPtr value_type; + inline static const char* GetTypeString (void*) { return "OffsetPtr"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return true; } + inline static bool AllowTransferOptimization () { return false; } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + if(IsSameType::result) + { + ReduceCopyData reduce; + transfer.template TransferPtr(!data.IsNull(), &reduce); + if (!data.IsNull()) + { + transfer.Transfer(*data, "data"); + } + transfer.template ReduceCopy (reduce); + } + else if(transfer.IsReading () || transfer.IsWriting ()) + { + bool isNull = data.IsNull(); + + transfer.template TransferPtr(true, NULL); + if (isNull) + { + mecanim::memory::ChainedAllocator* allocator = static_cast (transfer.GetUserData()); + data = allocator->Construct(); + } + + transfer.Transfer(*data, "data"); + } + else if(IsSameType::result) + { + transfer.template TransferPtr(false, NULL); + } + // Support for ProxyTransfer + else + { + transfer.template TransferPtr(false, NULL); + + TYPE p; + transfer.Transfer(p, "data"); + } + } +}; + +template +struct OffsetPtrArrayTransfer +{ + typedef T* iterator; + typedef T value_type; + + OffsetPtr& m_Data; + UInt32& m_ArraySize; + void* m_Allocator; + bool m_ClearPtrs; + + OffsetPtrArrayTransfer (OffsetPtr& data, UInt32& size, void* allocator, bool clearPtrs) + : m_Data(data),m_ArraySize(size) + { + m_Allocator = allocator; + m_ClearPtrs = clearPtrs; + } + + T* begin () { return m_Data.Get(); } + T* end () { return m_Data.Get() + m_ArraySize; } + size_t size() { return m_ArraySize; } + + void resize (int newSize) + { + m_ArraySize = newSize; + + mecanim::memory::ChainedAllocator* allocator = static_cast (m_Allocator); + Assert(allocator != NULL); + + if (newSize != 0) + { + m_Data = allocator->ConstructArray (newSize); + if (m_ClearPtrs) + memset(begin(), 0, sizeof(value_type) * newSize); + } + else + m_Data = NULL; + } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ +public: + + typedef OffsetPtrArrayTransfer value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + ReduceCopyData reduceCopy; + transfer.template TransferPtr(transfer.IsReading() || transfer.IsWriting() ? data.m_ArraySize != 0 : false, &reduceCopy); + + transfer.TransferSTLStyleArray (data); + transfer.template ReduceCopy(reduceCopy); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) + { + data.resize(rs); + } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + } +}; + + +#define MANUAL_ARRAY_TRANSFER2(TYPE,DATA,SIZE) OffsetPtrArrayTransfer DATA##ArrayTransfer (DATA, SIZE, transfer.GetUserData(), false); transfer.Transfer(DATA##ArrayTransfer, #DATA); +#define TRANSFER_BLOB_ONLY(DATA) if (IsSameType::result || IsSameType::result) transfer.Transfer(DATA, #DATA); + diff --git a/Runtime/Serialize/BuildTargetVerification.h b/Runtime/Serialize/BuildTargetVerification.h new file mode 100644 index 0000000..5bfdb2f --- /dev/null +++ b/Runtime/Serialize/BuildTargetVerification.h @@ -0,0 +1,78 @@ +#ifndef BUILDTARGETVERIFICATION_H +#define BUILDTARGETVERIFICATION_H + +#include "SerializationMetaFlags.h" +#include "Configuration/UnityConfigure.h" + +inline bool IsPCStandaloneTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return + // We don't support building for these any more, but we still need the constants for asset bundle + // backwards compatibility. + targetPlatform == kBuildStandaloneOSXUniversal || + targetPlatform == kBuildStandaloneOSXPPC || + + targetPlatform == kBuildStandaloneOSXIntel || + targetPlatform == kBuildStandaloneOSXIntel64 || + targetPlatform == kBuildStandaloneWinPlayer || + targetPlatform == kBuildStandaloneWin64Player || + targetPlatform == kBuildMetroPlayerX86 || + targetPlatform == kBuildMetroPlayerX64 || + targetPlatform == kBuildMetroPlayerARM || + targetPlatform == kBuildStandaloneLinux || + targetPlatform == kBuildStandaloneLinux64 || + targetPlatform == kBuildStandaloneLinuxUniversal || + targetPlatform == kBuildWinGLESEmu; +} + +inline bool IsWebPlayerTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return targetPlatform == kBuildWebPlayerLZMA || targetPlatform == kBuildWebPlayerLZMAStreamed || targetPlatform == kBuildNaCl; +} + +inline bool IsMetroTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return targetPlatform == kBuildMetroPlayerX86 || targetPlatform == kBuildMetroPlayerX64 || targetPlatform == kBuildMetroPlayerARM; +} + +inline bool CanLoadFileBuiltForTargetPlatform(BuildTargetPlatform targetPlatform) +{ + // Editor and BinaryToTextFile can load anything +#if UNITY_EDITOR || UNITY_EXTERNAL_TOOL + return true; + // Web players can handle all web formats +#elif WEBPLUG + return IsWebPlayerTargetPlatform(targetPlatform) || IsPCStandaloneTargetPlatform(targetPlatform); +#elif UNITY_METRO + // !! this code should be before #elif UNITY_WIN, because on Metro UNITY_WIN is defined as well !! + return IsMetroTargetPlatform(targetPlatform); +#elif UNITY_WP8 + // !! this code should be before #elif UNITY_WIN, because on WP8 UNITY_WIN is defined as well !! + return targetPlatform == kBuildWP8Player; +#elif UNITY_OSX || (UNITY_WIN && !UNITY_WINRT) || UNITY_LINUX + // Standalone can handle all web and standalone formats + return IsPCStandaloneTargetPlatform(targetPlatform) || IsWebPlayerTargetPlatform(targetPlatform); +#elif UNITY_WII + return targetPlatform == kBuildWii; +#elif UNITY_XENON + return targetPlatform == kBuildXBOX360; +#elif UNITY_PS3 + return targetPlatform == kBuildPS3; +#elif UNITY_IPHONE + return targetPlatform == kBuild_iPhone; +#elif UNITY_ANDROID + return targetPlatform == kBuild_Android; +#elif UNITY_FLASH + return targetPlatform == kBuildFlash; +#elif UNITY_WEBGL + return targetPlatform == kBuildWebGL; +#elif UNITY_BB10 + return targetPlatform == kBuildBB10; +#elif UNITY_TIZEN + return targetPlatform == kBuildTizen; +#else +#error Unknown platform +#endif +} + +#endif diff --git a/Runtime/Serialize/CacheWrap.cpp b/Runtime/Serialize/CacheWrap.cpp new file mode 100644 index 0000000..5ea4190 --- /dev/null +++ b/Runtime/Serialize/CacheWrap.cpp @@ -0,0 +1,461 @@ +#include "UnityPrefix.h" +#include "CacheWrap.h" +#include "FileCache.h" +#include "Runtime/Utilities/Utility.h" +#include +#if UNITY_EDITOR +#include "Editor/Src/GUIDPersistentManager.h" +#include "Runtime/Utilities/FileUtilities.h" +#endif + +#include "Runtime/Scripting/ScriptingUtility.h" + +using namespace std; + +#if UNITY_WINRT +#define OUTPUT_FIELDS_OF_LAST_SERIALIZABLE OutputFieldsOfLastSerializableObject(); +#else +#define OUTPUT_FIELDS_OF_LAST_SERIALIZABLE +#endif + +CachedReader::CachedReader () +{ + m_Cacher = 0; + m_Block = -1; + m_OutOfBoundsRead = false; + m_ActiveResourceImage = NULL; + #if CHECK_SERIALIZE_ALIGNMENT + m_CheckSerializeAlignment = true; + #endif +} + +void CachedReader::InitRead (CacheReaderBase& cacher, size_t position, size_t readSize) +{ + AssertIf (m_Block != -1); + m_Cacher = &cacher; + AssertIf (m_Cacher == NULL); + m_CacheSize = m_Cacher->GetCacheSize (); + m_Block = position / m_CacheSize; + m_MaximumPosition = position + readSize; + m_MinimumPosition = position; + + LockCacheBlockBounded (); + + SetPosition (position); +} + +void CachedReader::InitResourceImages (ResourceImageGroup& resourceImageGroup) +{ + Assert(m_ActiveResourceImage == NULL); + m_ResourceImageGroup = resourceImageGroup; +} + + +void CachedReader::LockCacheBlockBounded () +{ + m_Cacher->LockCacheBlock (m_Block, &m_CacheStart, &m_CacheEnd); + UInt8* maxPos = m_MaximumPosition - m_Block * m_CacheSize + m_CacheStart; + m_CacheEnd = min(m_CacheEnd, maxPos); +} + +size_t CachedReader::End () +{ + AssertIf (m_Block == -1); + size_t position = GetPosition (); + OutOfBoundsError (position, 0); + + m_Cacher->UnlockCacheBlock (m_Block); + m_Block = -1; + return position; +} + +void CachedReader::GetStreamingInfo (size_t offset, size_t size, StreamingInfo* streamingInfo) +{ + Assert (m_ActiveResourceImage != NULL || m_ActiveResourceImage == m_ResourceImageGroup.resourceImages[kStreamingResourceImage]); + + streamingInfo->offset = offset; + streamingInfo->size = size; + streamingInfo->path = m_ActiveResourceImage->GetStreamingPath(); +} + +CachedReader::~CachedReader () +{ + AssertIf (m_Block != -1); +} + +UInt8* CachedReader::FetchResourceImageData (size_t offset, size_t size) +{ + if (m_ActiveResourceImage == NULL) + { + ErrorString("Resource image for '" + m_Cacher->GetPathName() + "' couldn't be loaded!"); + return NULL; + } + + return m_ActiveResourceImage->Fetch (offset, size); +} + +ResourceImage::ResourceImage (const std::string& path, bool streaming) +{ + if (!streaming) + { + m_Size = GetFileLength(path); + m_Data = static_cast (UNITY_MALLOC(kMemResource, m_Size)); + + if (!ReadFromFile(path, m_Data, 0, m_Size)) + { + ErrorString("Resource image couldn't be loaded completely"); + } + } + else + { + m_StreamingPath = path; + } +} + +ResourceImage::~ResourceImage () +{ + if (m_Data) + UNITY_FREE(kMemResource, m_Data); +} + +void CachedReader::Read (void* data, size_t size) +{ + if (m_CachePosition + size <= m_CacheEnd) + { + memcpy (data, m_CachePosition, size); + m_CachePosition += size; + } + else + { + // Read some data directly if it is coming in big chunks and we are not hitting the end of the file! + size_t position = GetPosition (); + OutOfBoundsError (position, size); + + if (m_OutOfBoundsRead) + { + memset(data, 0, size); + return; + } + + // Read enough bytes from the cache to align the position with the cache size + if (position % m_CacheSize != 0) + { + int blockEnd = ((position / m_CacheSize) + 1) * m_CacheSize; + int curReadSize = min (size, blockEnd - position); + UpdateReadCache (data, curReadSize); + (UInt8*&)data += curReadSize; + size -= curReadSize; + position += curReadSize; + } + + // If we have a big block of data read directly without a cache, all aligned reads + int physicallyLimitedSize = min ((position + size), m_Cacher->GetFileLength ()) - position; + int blocksToRead = physicallyLimitedSize / m_CacheSize; + if (blocksToRead > 0) + { + int curReadSize = blocksToRead * m_CacheSize; + m_Cacher->DirectRead ((UInt8*)data, position, curReadSize); + m_CachePosition += curReadSize; + (UInt8*&)data += curReadSize; + size -= curReadSize; + } + + // Read the rest of the data from the cache! + while (size != 0) + { + int curReadSize = min (size, m_CacheSize); + UpdateReadCache (data, curReadSize); + (UInt8*&)data += curReadSize; + size -= curReadSize; + } + } +} + +void CachedReader::Skip (int size) +{ + if (m_CachePosition + size <= m_CacheEnd) + { + m_CachePosition += size; + } + else + { + int position = GetPosition (); + SetPosition(position + size); + } +} + +void memcpy_constrained_src (void* dst, const void* src, int size, const void* srcFrom, void* srcTo); +void memcpy_constrained_dst (void* dst, const void* src, int size, const void* dstFrom, void* dstTo); + +void memcpy_constrained_src (void* dst, const void* src, int size, const void* srcFrom, void* srcTo) +{ + UInt8* fromClamped = clamp ((UInt8*)src, (UInt8*)srcFrom, (UInt8*)srcTo); + UInt8* toClamped = clamp ((UInt8*)src + size, (UInt8*)srcFrom, (UInt8*)srcTo); + + int offset = fromClamped - (UInt8*)src; + size = toClamped - fromClamped; + memcpy ((UInt8*)dst + offset, (UInt8*)src + offset, size); +} + +void memcpy_constrained_dst (void* dst, const void* src, int size, const void* dstFrom, void* dstTo) +{ + UInt8* fromClamped = clamp ((UInt8*)dst, (UInt8*)dstFrom, (UInt8*)dstTo); + UInt8* toClamped = clamp ((UInt8*)dst + size, (UInt8*)dstFrom, (UInt8*)dstTo); + + int offset = fromClamped - (UInt8*)dst; + size = toClamped - fromClamped; + memcpy ((UInt8*)dst + offset, (UInt8*)src + offset, size); +} + +void CachedReader::UpdateReadCache (void* data, size_t size) +{ + AssertIf (m_Cacher == NULL); + AssertIf (size > m_CacheSize); + + size_t position = GetPosition (); + OutOfBoundsError(position, size); + + if (m_OutOfBoundsRead) + { + memset(data, 0, size); + return; + } + + // copy data oldblock + SetPosition (position); + memcpy_constrained_src (data, m_CachePosition, size, m_CacheStart, m_CacheEnd); + + // Read next cache block only if we actually need it. + if (m_CachePosition + size > m_CacheEnd) + { + // Check if the cache block + // copy data new block + SetPosition (position + size); + UInt8* cachePosition = position - m_Block * m_CacheSize + m_CacheStart; + memcpy_constrained_src (data, cachePosition, size, m_CacheStart, m_CacheEnd); + } + else + { + m_CachePosition += size; + } +} + +std::string GetNicePath (const CacheReaderBase& cacher) +{ + #if UNITY_EDITOR + string path = cacher.GetPathName(); + string assetPath = AssetPathNameFromAnySerializedPath(GetProjectRelativePath(cacher.GetPathName())); + + if (!assetPath.empty()) + return cacher.GetPathName() + "\' - \'" + assetPath; + else + return cacher.GetPathName(); + #else + return cacher.GetPathName(); + #endif +} + +void CachedReader::OutOfBoundsError (size_t position, size_t size) +{ + if (m_OutOfBoundsRead) + return; + + #define ERROR_FMT "The file \'%s\' is corrupted! Remove it and launch unity again!\n" \ + "[Position out of bounds! %" PRINTF_SIZET_FORMAT " %s %" PRINTF_SIZET_FORMAT "]" + + if (position + size > m_Cacher->GetFileLength ()) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, ">", m_Cacher->GetFileLength ()); + m_OutOfBoundsRead = true; + } + + if (position + size > m_MaximumPosition) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, ">", m_MaximumPosition); + m_OutOfBoundsRead = true; + } + + if (position < m_MinimumPosition) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, "<", m_MinimumPosition); + m_OutOfBoundsRead = true; + } +} + +void CachedReader::SetPosition (size_t position) +{ + OutOfBoundsError(position, 0); + if (m_OutOfBoundsRead) + return; + + if (position / m_CacheSize != m_Block) + { + m_Cacher->UnlockCacheBlock (m_Block); + m_Block = position / m_CacheSize; + m_Cacher->LockCacheBlock (m_Block, &m_CacheStart, &m_CacheEnd); + } + m_CachePosition = position - m_Block * m_CacheSize + m_CacheStart; +} + +void CachedReader::Align4Read () +{ + UInt32 offset = m_CachePosition - m_CacheStart; + offset = ((offset + 3) >> 2) << 2; + m_CachePosition = m_CacheStart + offset; +} + + +////// + +void CachedWriter::InitActiveWriter (ActiveWriter& activeWriter, CacheWriterBase& cacher) +{ + Assert (activeWriter.block == -1); + Assert (&cacher != NULL); + + activeWriter.cacheBase = &cacher; + activeWriter.block = 0; + activeWriter.cacheBase->LockCacheBlock (activeWriter.block, &activeWriter.cacheStart, &activeWriter.cacheEnd); + activeWriter.cachePosition = activeWriter.cacheStart; +} + +void CachedWriter::Align4Write () +{ + UInt32 leftOver = Align4LeftOver (m_ActiveWriter.cachePosition - m_ActiveWriter.cacheStart); + UInt8 value = 0; + for (UInt32 i=0;iGetCacheSize()); + UpdateWriteCache (data, curWriteSize); + (UInt8*&)data += curWriteSize; + size -= curWriteSize; + } + } +} + +void CachedWriter::UpdateWriteCache (const void* data, size_t size) +{ + Assert (m_ActiveWriter.cacheBase != NULL); + AssertIf (size > m_ActiveWriter.cacheBase->GetCacheSize()); + + size_t position = GetPosition (); + size_t cacheSize = m_ActiveWriter.cacheBase->GetCacheSize(); + // copy data from oldblock + memcpy_constrained_dst (m_ActiveWriter.cachePosition, data, size, m_ActiveWriter.cacheStart, m_ActiveWriter.cacheEnd); + + SetPosition (position + size); + + // copy data new block + UInt8* cachePosition = position - m_ActiveWriter.block * cacheSize + m_ActiveWriter.cacheStart; + memcpy_constrained_dst (cachePosition, data, size, m_ActiveWriter.cacheStart, m_ActiveWriter.cacheEnd); +} + +void CachedWriter::SetPosition (size_t position) +{ + size_t cacheSize = m_ActiveWriter.cacheBase->GetCacheSize(); + int newBlock = position / cacheSize; + if (newBlock != m_ActiveWriter.block) + { + Assert(newBlock == m_ActiveWriter.block + 1); + + m_ActiveWriter.cacheBase->UnlockCacheBlock (m_ActiveWriter.block); + m_ActiveWriter.block = newBlock; + m_ActiveWriter.cacheBase->LockCacheBlock (m_ActiveWriter.block, &m_ActiveWriter.cacheStart, &m_ActiveWriter.cacheEnd); + } + m_ActiveWriter.cachePosition = position - m_ActiveWriter.block * cacheSize + m_ActiveWriter.cacheStart; +} + +size_t CachedWriter::GetPosition () const +{ + return m_ActiveWriter.GetPosition(); +} + + +bool CachedWriter::CompleteWriting () +{ + m_ActiveWriter.cacheBase->UnlockCacheBlock (m_ActiveWriter.block); + + bool success = m_ActiveWriter.cacheBase->CompleteWriting (m_ActiveWriter.GetPosition()); + + #if UNITY_EDITOR + if (m_ActiveResourceImageMode != kResourceImageNotSupported) + { + for (int i=0;iCompleteWriting (m_ResourceImageWriters[i].GetPosition()); + success &= m_ResourceImageWriters[i].cacheBase->WriteHeaderAndCloseFile (NULL, 0, 0); + } + } + #endif + + return success; +} + +size_t CachedWriter::ActiveWriter::GetPosition () const +{ + return cachePosition - cacheStart + block * cacheBase->GetCacheSize(); +} + +#if UNITY_EDITOR + +void CachedWriter::InitWrite (CacheWriterBase& cacher) +{ + InitActiveWriter(m_ActiveWriter, cacher); + m_DefaultWriter = m_ActiveWriter; + m_ActiveResourceImageMode = kResourceImageNotSupported; +} + +void CachedWriter::InitResourceImage (ActiveResourceImage index, CacheWriterBase& resourceImage) +{ + m_ActiveResourceImageMode = kResourceImageInactive; + InitActiveWriter(m_ResourceImageWriters[index], resourceImage); +} + +void CachedWriter::BeginResourceImage (ActiveResourceImage resourceImage) +{ + if (m_ActiveResourceImageMode == kResourceImageNotSupported) + return; + + Assert(m_ActiveResourceImageMode == kResourceImageInactive); + Assert(resourceImage > kResourceImageInactive); + + m_ActiveResourceImageMode = resourceImage; + + m_DefaultWriter = m_ActiveWriter; + m_ActiveWriter = m_ResourceImageWriters[m_ActiveResourceImageMode]; + Assert(m_ActiveWriter.cacheBase != NULL); +} + +void CachedWriter::EndResourceImage () +{ + Assert(IsWritingResourceImage()); + + m_ResourceImageWriters[m_ActiveResourceImageMode] = m_ActiveWriter; + m_ActiveWriter = m_DefaultWriter; + + m_ActiveResourceImageMode = kResourceImageInactive; +} + +#else + +void CachedWriter::InitWrite (CacheWriterBase& cacher) +{ + InitActiveWriter(m_ActiveWriter, cacher); +} + +#endif diff --git a/Runtime/Serialize/CacheWrap.h b/Runtime/Serialize/CacheWrap.h new file mode 100644 index 0000000..6ded041 --- /dev/null +++ b/Runtime/Serialize/CacheWrap.h @@ -0,0 +1,229 @@ +#ifndef CACHEWRAP_H +#define CACHEWRAP_H + +#include "Configuration/UnityConfigure.h" +#include "Runtime/Serialize/FileCache.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + +class CachedWriter +{ + struct ActiveWriter + { + UInt8* cachePosition; + UInt8* cacheStart; + UInt8* cacheEnd; + SInt32 block; + CacheWriterBase* cacheBase; + + ActiveWriter () { cachePosition = NULL; cacheStart = NULL; cacheEnd = NULL; block = -1; cacheBase = NULL; } + size_t GetPosition () const; + }; + + ActiveWriter m_ActiveWriter; + + #if UNITY_EDITOR + + ActiveResourceImage m_ActiveResourceImageMode; + + ActiveWriter m_DefaultWriter; + ActiveWriter m_ResourceImageWriters[kNbResourceImages]; + + #endif + + static void InitActiveWriter (CachedWriter::ActiveWriter& activeWriter, CacheWriterBase& cacher); + void SetPosition (size_t position); + void EXPORT_COREMODULE UpdateWriteCache (const void* data, size_t size); + + public: + + void InitWrite (CacheWriterBase& cacher); + + bool CompleteWriting (); + + +#if UNITY_EDITOR + void InitResourceImage (ActiveResourceImage index, CacheWriterBase& resourceImage); + + void BeginResourceImage (ActiveResourceImage resourceImageType); + void EndResourceImage (); + bool IsWritingResourceImage () { return m_ActiveResourceImageMode > kResourceImageInactive; } + + CacheWriterBase& GetCacheBase () { return *m_ActiveWriter.cacheBase; } + +#endif + + template + void Write (const T& data) + { +#if CHECK_SERIALIZE_ALIGNMENT + if (m_CheckSerializeAlignment) + { + SInt32 position = reinterpret_cast(m_ActiveWriter.cachePosition); + SInt32 size = sizeof(T); + SInt32 align = position % size; + if (align != 0) + { + ErrorString("Alignment error "); + } + } +#endif + + if (m_ActiveWriter.cachePosition + sizeof (T) < m_ActiveWriter.cacheEnd) + { + *reinterpret_cast (m_ActiveWriter.cachePosition) = data; + m_ActiveWriter.cachePosition += sizeof (T); + } + else + UpdateWriteCache (&data, sizeof (data)); + } + + void Align4Write (); + + void Write (const void* data, size_t size); + + size_t GetPosition () const; +}; + +struct StreamingInfo +{ + size_t offset; + size_t size; + std::string path; + + bool IsValid () const { return !path.empty(); } + + StreamingInfo () { offset = 0; size = 0; } +}; + + +struct ResourceImage +{ + UInt8* m_Data; + UInt32 m_Size; + std::string m_StreamingPath; + + public: + + ResourceImage (const std::string& path, bool stream); + ~ResourceImage (); + + UInt8* Fetch (size_t offset, size_t size) + { + Assert(m_Data != NULL); + Assert(size + offset <= m_Size); + return m_Data + offset; + } + + const std::string& GetStreamingPath () { Assert(!m_StreamingPath.empty()); return m_StreamingPath; } +}; + +struct EXPORT_COREMODULE ResourceImageGroup +{ + ResourceImage* resourceImages[kNbResourceImages]; + + ResourceImageGroup () { memset(this, 0, sizeof(ResourceImageGroup)); } +}; + +class EXPORT_COREMODULE CachedReader +{ + private: + + UInt8* m_CachePosition; + UInt8* m_CacheStart; + UInt8* m_CacheEnd; + CacheReaderBase* m_Cacher; + SInt32 m_Block; + size_t m_CacheSize; + size_t m_MinimumPosition; + size_t m_MaximumPosition; + bool m_OutOfBoundsRead; + + ResourceImage* m_ActiveResourceImage; + ResourceImageGroup m_ResourceImageGroup; + + void UpdateReadCache (void* data, size_t size); + + CachedReader (const CachedReader& c);// undefined + CachedReader& operator = (const CachedReader& c);// undefined + + void OutOfBoundsError (size_t position, size_t size); + void LockCacheBlockBounded(); + + public: + + CachedReader (); + ~CachedReader (); + + void InitRead (CacheReaderBase& cacher, size_t position, size_t size); + void InitResourceImages (ResourceImageGroup& resourceImage); + + size_t GetEndPosition () { return m_MaximumPosition; } + + size_t End (); + + template + void Skip () + { + m_CachePosition += sizeof (T); + } + + void Skip (int size); + + UInt8* FetchResourceImageData (size_t offset, size_t size); + + void GetStreamingInfo (size_t offset, size_t size, StreamingInfo* streamingInfo); + + void BeginResourceImage (ActiveResourceImage index) { m_ActiveResourceImage = m_ResourceImageGroup.resourceImages[index]; } + void EndResourceImage () { m_ActiveResourceImage = NULL; } + bool IsReadingResourceImage () { return m_ActiveResourceImage != NULL; } + const char* GetSerializedFilePathName() { return m_Cacher->GetPathName().c_str(); } + + template + void Read (T& data, size_t position) + { + m_CachePosition = m_CacheStart + position - m_Block * m_CacheSize; + if (m_CachePosition >= m_CacheStart && m_CachePosition + sizeof (data) <= m_CacheEnd) + { + data = *reinterpret_cast (m_CachePosition); + m_CachePosition += sizeof (T); + } + else + UpdateReadCache (&data, sizeof (data)); + } + + template + void Read (T& data) + { + if (m_CachePosition + sizeof (T) <= m_CacheEnd) + { + data = *reinterpret_cast (m_CachePosition); + m_CachePosition += sizeof (T); + } + else + UpdateReadCache (&data, sizeof (data)); + } + + void Align4Read (); + + size_t GetPosition () const { return m_CachePosition - m_CacheStart + m_Block * m_CacheSize; } + void SetPosition (size_t position); + void SetAbsoluteMemoryPosition(UInt8* position) { m_CachePosition = position; } + UInt8* GetAbsoluteMemoryPosition() { return m_CachePosition; } + + void Read (void* data, size_t size); + + CacheReaderBase* GetCacher () const { return m_Cacher; } +}; + +inline UInt32 Align4 (UInt32 size) +{ + UInt32 value = ((size + 3) >> 2) << 2; + return value; +} + +inline UInt32 Align4LeftOver (UInt32 size) +{ + return Align4(size) - size; +} + +#endif diff --git a/Runtime/Serialize/DumpSerializedDataToText.cpp b/Runtime/Serialize/DumpSerializedDataToText.cpp new file mode 100644 index 0000000..1be6ddf --- /dev/null +++ b/Runtime/Serialize/DumpSerializedDataToText.cpp @@ -0,0 +1,372 @@ +#include "UnityPrefix.h" +#include "DumpSerializedDataToText.h" +#include "TypeTree.h" +#include "CacheWrap.h" +#include "Runtime/Utilities/GUID.h" + +using namespace std; + +#if (UNITY_EDITOR || UNITY_INCLUDE_SERIALIZATION_DUMP || DEBUGMODE) + +void DumpSerializedDataToText (const TypeTree& typeTree, dynamic_array& data) +{ + int offset = 0; + RecursiveOutput(typeTree, data.begin(), &offset, 0, cout, kDumpNormal, 0, false, -1); +} +#define TAB for (int t=0;t +void DoSwap (T& t, bool swapBytes) +{ + if (swapBytes) + SwapEndianBytes(t); +} + + +SInt32 CalculateByteSize(const TypeTree& type) { + if(type.m_ByteSize != -1) + return type.m_ByteSize; + + SInt32 r=0; + for (TypeTree::const_iterator i=type.begin ();i != type.end ();i++) + r+=CalculateByteSize(*i); + return r; +} + +void OutputType (const TypeTree& type, ostream& os) +{ + os << "Name: " << type.m_Name; + os << " Type: " << type.m_Type; + os << " ByteSize: " << type.m_ByteSize; + os << " TypeTreePosition: " << type.m_Index; + os << " IsArray: " << type.m_IsArray; + os << " Version: " << type.m_Version; + os << " MetaFlag: " << type.m_MetaFlag; + os << " IsArray: " << type.m_IsArray; +} + +string ExtractString (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + string value; + SInt32 size = *reinterpret_cast (data + *offset); + DoSwap(size, doSwap); + value.reserve (size); + for (int i=0;i (data + *offset); + DoSwap(size, doSwap); + value.reserve (size*2); + for (int i=0;i (data + *offset); + DoSwap(v, doSwap); + if (i != 0) + val += ' '; + val += Format("%g", v); + *offset += 4; + } + val += ')'; + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return val; +} + +string ExtractRectOffset (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + AssertIf(type.m_Father == NULL); + AssertIf(type.m_Children.size() != 4); + + string val = "("; + TypeTree::TypeTreeList::const_iterator it = type.m_Children.begin(); + for (int i = 0; i < 4; ++i, ++it) + { + int v = *reinterpret_cast (data + *offset); + DoSwap(v, doSwap); + if (i != 0) + val += ' '; + val += Format("%s %i", it->m_Name.c_str(), v); + *offset += 4; + } + val += ')'; + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return val; +} + + +string ExtractPPtr (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + SInt32 fileID = *reinterpret_cast(data + *offset); + SInt32 pathID = *reinterpret_cast(data + *offset + 4); + DoSwap(fileID, doSwap); + DoSwap(pathID, doSwap); + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + *offset += 8; + + return Format ("(file %i path %i)", (int)fileID, (int)pathID); +} + +string ExtractGUID (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + AssertIf(type.m_Father == NULL); + AssertIf(type.m_Children.size() != 4); + AssertIf(CalculateByteSize(type) != 4*4); + + UnityGUID val; + for (int i = 0; i < 4; ++i) { + UInt32 v = *reinterpret_cast (data + *offset); + val.data[i]=v; + *offset += 4; + } + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return GUIDToString(val); +} + + +void OutputValue (const TypeTree& type, const UInt8* data, int* offset, ostream& os, bool doSwap) +{ +#define OUTPUT(x) \ +else if (type.m_Type == #x) \ +{ \ +x value = *reinterpret_cast (data + *offset); \ +DoSwap(value, doSwap);\ +os << value; \ +} + +#define OUTPUT_INT(x) \ +else if (type.m_Type == #x) \ +{ \ +x value = *reinterpret_cast (data + *offset); \ +DoSwap(value, doSwap);\ +int intValue = value; \ +os << intValue; \ +} + + + if (false) { } + OUTPUT (float) + OUTPUT (double) + OUTPUT (int) + OUTPUT (unsigned int) + OUTPUT (SInt32) + OUTPUT (UInt32) + OUTPUT (SInt16) + OUTPUT (UInt16) + OUTPUT (SInt64) + OUTPUT (UInt64) + OUTPUT_INT (SInt8) + OUTPUT_INT (UInt8) + OUTPUT (char) + OUTPUT (bool) + else + { + AssertString ("Unsupported type! " + type.m_Type); + } + *offset = *offset + type.m_ByteSize; +} + +const int kArrayMemberColumns = 25; + +void RecursiveOutput (const TypeTree& type, const UInt8* data, int* offset, int tab, ostream& os, DumpOutputMode mode, int pathID, bool doSwap, int arrayIndex) +{ + if (type.m_Type == "Vector3f" && type.m_ByteSize != 12) + { + AssertString ("Unsupported type! " + type.m_Type); + } + + if (!type.m_IsArray && arrayIndex == -1 && mode != kDumpClean) + os << Format("% 5d: ", (int)type.m_Index); + + if (type.IsBasicDataType ()) + { + // basic data type + if (arrayIndex == -1) + { + TAB os << type.m_Name << " "; + OutputValue (type, data, offset, os, doSwap); + os << " (" << type.m_Type << ")"; + os << endl; + } + else + { + // array members: multiple members per line + if (arrayIndex % kArrayMemberColumns == 0) + { + if (arrayIndex != 0) + os << endl; + TAB os << type.m_Name << " (" << type.m_Type << ") #" << arrayIndex << ": "; + OutputValue (type, data, offset, os, doSwap); + } + else + { + os << ' '; + OutputValue (type, data, offset, os, doSwap); + } + } + } + else if (type.m_IsArray) + { + // Extract and Print size + int size = *reinterpret_cast (data + *offset); + DoSwap(size, doSwap); + + RecursiveOutput (type.m_Children.front (), data, offset, tab, os, mode, 0, doSwap, -1); + // Print children + for (int i=0;i +#include "Runtime/Utilities/dynamic_array.h" + +class TypeTree; + +enum DumpOutputMode +{ + kDumpNormal, + kDumpClean, +}; + +void DumpSerializedDataToText (const TypeTree& typeTree, dynamic_array& data); +void RecursiveOutput (const TypeTree& type, const UInt8* data, int* offset, int tab, std::ostream& os, DumpOutputMode mode, int pathID, bool doSwap, int arrayIndex); +#endif + diff --git a/Runtime/Serialize/FileCache.cpp b/Runtime/Serialize/FileCache.cpp new file mode 100644 index 0000000..c297ed6 --- /dev/null +++ b/Runtime/Serialize/FileCache.cpp @@ -0,0 +1,461 @@ +#include "UnityPrefix.h" +#include "FileCache.h" +#include "Runtime/Utilities/algorithm_utility.h" +#include "Runtime/Utilities/Utility.h" +#include "Runtime/Utilities/PathNameUtility.h" + +#if SUPPORT_SERIALIZE_WRITE +#include "Runtime/Utilities/FileUtilities.h" +#endif + +using namespace std; + +#define USE_OPEN_FILE_CACHE UNITY_EDITOR + +#if USE_OPEN_FILE_CACHE + +struct OpenFilesCache +{ + enum { kOpenedFileCacheCount = 5 }; + File* m_Cache[kOpenedFileCacheCount]; + UInt32 m_TimeStamps[kOpenedFileCacheCount]; + UInt32 m_TimeStamp; + + OpenFilesCache () + { + m_TimeStamp = 0; + for (int i=0;iLock (File::kNone, false); + #endif + m_Cache[lruIndex]->Close (); + } + + m_Cache[lruIndex] = theFile; + m_TimeStamps[lruIndex] = m_TimeStamp; + + if(!theFile->Open (thePath, File::kReadPermission, File::kSilentReturnOnOpenFail)) + ErrorString(Format("Could not open file %s for read", thePath.c_str())); + + #if UNITY_OSX + theFile->Lock (File::kShared, false); + #endif + } + + void ForceCloseAll () + { + // Find and close cache + for (int i=0;iLock (File::kNone, false); + #endif + m_Cache[i]->Close (); + m_Cache[i] = NULL; + m_TimeStamps[i] = 0; + return; + } + } + } +}; +OpenFilesCache gOpenFilesCache; + +void ForceCloseAllOpenFileCaches () +{ + gOpenFilesCache.ForceCloseAll (); +} + +#endif + + +CacheReaderBase::~CacheReaderBase () +{} + +CacheWriterBase::~CacheWriterBase () +{} + +FileCacherRead::FileCacherRead (const string& pathName, size_t cacheSize, size_t cacheCount) +{ + m_RootHeader = GET_CURRENT_ALLOC_ROOT_HEADER(); + m_Path = PathToAbsolutePath(pathName); + + // Initialize Cache + m_MaxCacheCount = cacheCount; + m_CacheSize = cacheSize; + m_TimeStamp = 0; + + // Get File size + m_FileSize = ::GetFileLength(m_Path); + + #if !USE_OPEN_FILE_CACHE + if(!m_File.Open(m_Path, File::kReadPermission, File::kSilentReturnOnOpenFail)) + ErrorString(Format("Could not open file %s for read", m_Path.c_str())); + + // Make the file non-overwritable by the cache (to make caching behaviour imitate windows) + #if UNITY_OSX + m_File.Lock (File::kShared, false); + #endif + + #endif + #if DEBUG_LINEAR_FILE_ACCESS + m_LastFileAccessPosition = 0; + #endif +} + +FileCacherRead::~FileCacherRead () +{ + for (CacheBlocks::iterator i = m_CacheBlocks.begin ();i != m_CacheBlocks.end ();++i) + { + AssertIf (i->second.lockCount); + UNITY_FREE(kMemFile,i->second.data); + } + + m_CacheBlocks.clear (); + + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.ForceClose (&m_File); + #else + // Make the file overwritable by the cache. + #if UNITY_OSX + m_File.Lock (File::kNone, false); + #endif + + m_File.Close(); + + #endif +} + +void FileCacherRead::ReadCacheBlock (CacheBlock& cacheBlock) +{ + int block = cacheBlock.block; + // Watch out for not reading over eof + int readSize = min (m_CacheSize, m_FileSize - block * m_CacheSize); + + // load the data from disk + // only if the physical file contains any data for this block + if (readSize > 0) + { + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.OpenCached (&m_File, m_Path); + #endif + + #if DEBUG_LINEAR_FILE_ACCESS + size_t position = block * m_CacheSize; + #endif + + m_File.Read (block * m_CacheSize, cacheBlock.data, readSize); + + #if DEBUG_LINEAR_FILE_ACCESS + // printf_console("ACCESS: [%08x] %s %i bytes @ %i to %08x\n",this, __FUNCTION__, readSize, block * m_CacheSize, cacheBlock.data); + std::string fileName = GetLastPathNameComponent(m_Path); + if (position < m_LastFileAccessPosition && fileName != "mainData" && fileName != "unity default resources") + { + ErrorString(Format("File access: %s is not linear Reading: %d Seek position: %d", fileName.c_str(), position, m_LastFileAccessPosition)); + } + + m_LastFileAccessPosition = position + readSize; + #endif + } +} + +void FileCacherRead::DirectRead (void* data, size_t position, size_t size) +{ + // load the data from disk + // only if the physical file contains any data for this block + FatalErrorIf (m_FileSize - position < size); + + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.OpenCached (&m_File, m_Path); + #endif + + m_File.Read (position, data, size); + + #if DEBUG_LINEAR_FILE_ACCESS + // printf_console("ACCCESS: [%08x] %s %i bytes @ %i to %08x\n",this, __FUNCTION__, size, position, data); + std::string fileName = GetLastPathNameComponent(m_Path); + if (position < m_LastFileAccessPosition && fileName != "mainData" && fileName != "unity default resources") + { + ErrorString(Format("File access: %s is not linear Reading: %d Seek position: %d", fileName.c_str(), position, m_LastFileAccessPosition)); + } + + m_LastFileAccessPosition = position + size; + #endif +} + + +bool FileCacherRead::FreeSingleCache () +{ + unsigned lowestTimeStamp = -1; + CacheBlocks::iterator lowestCacheBlock = m_CacheBlocks.end (); + + CacheBlocks::iterator i; + for (i=m_CacheBlocks.begin ();i != m_CacheBlocks.end ();i++) + { + if (i->second.lockCount == 0) + { + if (i->second.timeStamp < lowestTimeStamp) + { + lowestTimeStamp = i->second.timeStamp; + lowestCacheBlock = i; + } + } + } + + if (lowestCacheBlock != m_CacheBlocks.end ()) + { + CacheBlock& block = lowestCacheBlock->second; + UNITY_FREE(kMemFile,block.data); + m_CacheBlocks.erase (lowestCacheBlock); + return true; + } + else + return false; +} + +FileCacherRead::CacheBlock& FileCacherRead::AllocateCacheBlock (int block) +{ + AssertIf (m_CacheBlocks.count (block)); + CacheBlock cacheBlock; + cacheBlock.block = block; + cacheBlock.lockCount = 0; + cacheBlock.timeStamp = 0; + cacheBlock.data = (UInt8*)UNITY_MALLOC(MemLabelId(kMemFileId, m_RootHeader),m_CacheSize); + m_CacheBlocks[block] = cacheBlock; + return m_CacheBlocks[block]; +} + +void FileCacherRead::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + CacheBlock* newCacheBlock; + CacheBlocks::iterator i = m_CacheBlocks.find (block); + if (i != m_CacheBlocks.end ()) + newCacheBlock = &i->second; + else + { + // Make room for a new cache block + if (m_CacheBlocks.size () >= m_MaxCacheCount) + FreeSingleCache (); + + newCacheBlock = &AllocateCacheBlock (block); + ReadCacheBlock (*newCacheBlock); + } + AssertIf (newCacheBlock->block != block); + + newCacheBlock->timeStamp = ++m_TimeStamp; + newCacheBlock->lockCount++; + + *startPos = newCacheBlock->data; + *endPos = newCacheBlock->data + min (m_FileSize - block * GetCacheSize (), GetCacheSize ()); +} + +void FileCacherRead::UnlockCacheBlock (int block) +{ + AssertIf (!m_CacheBlocks.count (block)); + CacheBlock& cacheBlock = m_CacheBlocks.find (block)->second; + cacheBlock.lockCount--; + if (cacheBlock.lockCount == 0) + { + // LockCacheBlock sometimes locks more caches than m_MaxCacheCount + // Thus we should Free them as soon as they become unlocked + if (m_CacheBlocks.size () > m_MaxCacheCount) + FreeSingleCache (); + } +} + +std::string FileCacherRead::GetPathName() const +{ + return m_Path; +} + +#if SUPPORT_SERIALIZE_WRITE +FileCacherWrite::FileCacherWrite () +{ + m_Success = true; + m_Block = -1; + m_Locked = false; + m_CacheSize = 0; + m_DataCache = NULL; +} + +void FileCacherWrite::InitWriteFile (const std::string& pathName, size_t cacheSize) +{ + m_Path = PathToAbsolutePath(pathName); + m_Success = true; + + m_File.Open(m_Path, File::kWritePermission); + // file we're writing to always is a temporary, non-indexable file + SetFileFlags(m_Path, kAllFileFlags, kFileFlagDontIndex|kFileFlagTemporary); + + m_Block = -1; + m_Locked = false; + m_CacheSize = cacheSize; + m_DataCache = (UInt8*)UNITY_MALLOC (kMemFile, m_CacheSize); +} + +bool FileCacherWrite::CompleteWriting (size_t size) +{ + Assert(m_Block != -1); + + size_t remainingData = size - (m_Block * m_CacheSize); + Assert(remainingData <= m_CacheSize); + + m_Success &= m_File.Write(m_Block * m_CacheSize, m_DataCache, remainingData); + + return m_Success; +} + +bool FileCacherWrite::WriteHeaderAndCloseFile (void* data, size_t position, size_t size) +{ + Assert(position == 0); + if (size != 0) + m_Success &= m_File.Write(position, data, size); + m_Success &= m_File.Close(); + + return m_Success; +} + + +FileCacherWrite::~FileCacherWrite() +{ + if (m_DataCache) + { + UNITY_FREE (kMemFile, m_DataCache); + m_DataCache = NULL; + } + + m_File.Close(); +} + +void FileCacherWrite::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + AssertIf (block == -1); + Assert (block == m_Block || m_Block+1 == block ); + Assert (!m_Locked); + + if (m_Block != block) + { + AssertIf (m_Locked != 0); + + if (m_Block != -1) + m_Success &= m_File.Write(m_DataCache, m_CacheSize); + + m_Block = block; + } + + *startPos = m_DataCache; + *endPos = m_DataCache + m_CacheSize; + m_Locked++; +} + +void FileCacherWrite::UnlockCacheBlock (int block) +{ + Assert (block == m_Block); + Assert (m_Locked); + + m_Locked = false; +} + + +std::string FileCacherWrite::GetPathName() const +{ + return m_Path; +} + +#endif // SUPPORT_SERIALIZE_WRITE + + +MemoryCacherReadBlocks::MemoryCacherReadBlocks (UInt8** blocks, int size, size_t cacheBlockSize) +: m_CacheBlockSize(cacheBlockSize) +, m_Memory(blocks) +, m_FileSize(size) +{ +} + + +MemoryCacherReadBlocks::~MemoryCacherReadBlocks () +{ +} + +void MemoryCacherReadBlocks::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + /// VERIFY OUT OF BOUNDS!!! ??? + AssertIf(block > m_FileSize / m_CacheBlockSize); + *startPos = m_Memory[block]; + *endPos = *startPos + min (GetFileLength () - block * m_CacheBlockSize, m_CacheBlockSize); +} + +void MemoryCacherReadBlocks::DirectRead (void* data, size_t position, size_t size) +{ + ReadFileCache(*this, data, position, size); +} + +void ReadFileCache (CacheReaderBase& cacher, void* data, size_t position, size_t size) +{ + UInt8 *cacheStart, *cacheEnd; + UInt8 *from, *fromClamped, *to, *toClamped; + + int block = position / cacher.GetCacheSize (); + int lastBlock = (position + size - 1) / cacher.GetCacheSize (); + + while (block <= lastBlock) + { + cacher.LockCacheBlock (block, &cacheStart, &cacheEnd); + + // copy data from oldblock and unlock + from = cacheStart + (position - block * cacher.GetCacheSize ()); + fromClamped = clamp (from, cacheStart, cacheEnd); + to = cacheStart + (position + size - block * cacher.GetCacheSize ()); + toClamped = clamp (to, cacheStart, cacheEnd); + memcpy ((UInt8*)data + (fromClamped - from), fromClamped, toClamped - fromClamped); + + cacher.UnlockCacheBlock (block); + block++; + } +} diff --git a/Runtime/Serialize/FileCache.h b/Runtime/Serialize/FileCache.h new file mode 100644 index 0000000..6705b44 --- /dev/null +++ b/Runtime/Serialize/FileCache.h @@ -0,0 +1,360 @@ +#ifndef FILECACHE_H +#define FILECACHE_H + +#include "Runtime/Utilities/LogAssert.h" +#include "Configuration/UnityConfigure.h" +#include "Runtime/Utilities/File.h" +#include +#include +#include +#include +#include "Runtime/Allocator/MemoryMacros.h" +#include "Runtime/Utilities/dynamic_array.h" + +#define DEBUG_LINEAR_FILE_ACCESS 0 + +using std::max; +using std::min; +using std::list; + +class EXPORT_COREMODULE CacheReaderBase +{ +public: + + enum + { + kImmediatePriority = 0, + kPreloadPriority = 1, + kLoadPriorityCount = 2 + }; + + enum + { + kUnloadPriority = 0, + kPreloadedPriority = 1, + kRequiredPriority = 2, + kUnloadPriorityCount = 3 + }; + + virtual ~CacheReaderBase () = 0; + + virtual void DirectRead (void* data, size_t position, size_t size) = 0; + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) = 0; + virtual void UnlockCacheBlock (int block) = 0; + + virtual size_t GetCacheSize () const = 0; + virtual std::string GetPathName() const = 0; + virtual size_t GetFileLength () const = 0; + virtual UInt8* GetAddressOfMemory() { ErrorString("GetAddressOfMemory called on CacheReaderBase which does not support it"); return NULL; } +}; + +class CacheWriterBase +{ +public: + virtual ~CacheWriterBase () = 0; + + virtual bool CompleteWriting (size_t size) = 0; + virtual bool WriteHeaderAndCloseFile (void* /*data*/, size_t /*position*/, size_t /*size*/) { AssertString("Only used for writing serialized files"); return false; } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) = 0; + virtual void UnlockCacheBlock (int block) = 0; + + virtual size_t GetCacheSize () const = 0; + virtual std::string GetPathName() const = 0; + virtual size_t GetFileLength () const = 0; + virtual UInt8* GetAddressOfMemory() { ErrorString("GetAddressOfMemory called on CacheWriterBase which does not support it"); return NULL; } +}; + +class MemoryCacheReader : public CacheReaderBase +{ +protected: + enum + { + kCacheSize = 256 + }; + + dynamic_array& m_Memory; + SInt32 m_LockCount; + +public: + MemoryCacheReader (dynamic_array& mem) : m_Memory (mem), m_LockCount(0) { } + virtual ~MemoryCacheReader () { AssertIf (m_LockCount != 0); } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) + { + *startPos = m_Memory.size() > block * kCacheSize ? &m_Memory[block * kCacheSize] : NULL; + *endPos = *startPos + min (GetFileLength () - block * kCacheSize, kCacheSize); + m_LockCount++; + } + + virtual void DirectRead (void* data, size_t position, size_t size) + { + AssertIf (m_Memory.size () - position < size); + memcpy (data, &m_Memory[position], size); + } + + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + + virtual size_t GetFileLength () const { return m_Memory.size (); } + virtual size_t GetCacheSize () const { return kCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } + virtual UInt8* GetAddressOfMemory() { return &m_Memory[0]; } +}; + +class MemoryCacheWriter : public CacheWriterBase +{ +protected: + enum + { + kCacheSize = 256 + }; + + dynamic_array& m_Memory; + SInt32 m_LockCount; + +public: + MemoryCacheWriter (dynamic_array& mem) : m_Memory (mem), m_LockCount(0) { } + virtual ~MemoryCacheWriter () { AssertIf (m_LockCount != 0); } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) + { + m_Memory.resize_uninitialized (max ((block + 1) * kCacheSize, m_Memory.size ()), true); + *startPos = &m_Memory[block * kCacheSize]; + *endPos = *startPos + kCacheSize; + + m_LockCount++; + } + + virtual bool CompleteWriting (size_t size) { m_Memory.resize_uninitialized (size); m_Memory.shrink_to_fit(); return true; } + + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + + virtual size_t GetFileLength () const { return m_Memory.size (); } + virtual size_t GetCacheSize () const { return kCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } +}; + +enum +{ + kBlockCacherCacheSize = 256 +}; + +class BlockMemoryCacheWriter : public CacheWriterBase +{ +protected: + + enum + { + kNumBlockReservations = 256 + }; + + size_t m_Size; + SInt32 m_LockCount; + MemLabelId m_AllocLabel; + + // It is possible to use the custom allocator for this index as well -- however, + // using the tracking linear tempory allocator is most efficient, when deallocating + // in the exact opposite order of allocating, which can only be guaranteed, when all allocations + // are in our control. + typedef UNITY_VECTOR(kMemFile, UInt8*) BlockVector; + BlockVector m_Blocks; + + public: + + BlockMemoryCacheWriter (MemLabelId label) + : m_AllocLabel(label) + , m_Blocks() + { + m_Blocks.reserve(kNumBlockReservations); + m_Size = 0; + m_LockCount = 0; + } + + ~BlockMemoryCacheWriter () { + AssertIf (m_LockCount != 0); + for(BlockVector::reverse_iterator i = m_Blocks.rbegin(); i != m_Blocks.rend(); i++) + UNITY_FREE(m_AllocLabel, *i); + } + + void ResizeBlocks (int newBlockSize) + { + int oldBlockSize = m_Blocks.size(); + + for(int block = oldBlockSize-1; block>=newBlockSize; block--) + UNITY_FREE(m_AllocLabel, m_Blocks[block]); + + if(m_Blocks.capacity() < newBlockSize) + m_Blocks.reserve(m_Blocks.capacity() * 2); + + m_Blocks.resize(newBlockSize); + + for(int block = oldBlockSize; block(block+1, m_Blocks.size())); + *startPos = m_Blocks[block]; + *endPos = *startPos + kBlockCacherCacheSize; + m_LockCount++; + } + + virtual bool CompleteWriting (size_t size) + { + m_Size = size; + ResizeBlocks(m_Size/kBlockCacherCacheSize + 1); + return true; + } + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + virtual size_t GetFileLength () const { return m_Size; } + virtual size_t GetCacheSize () const { return kBlockCacherCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } + + // Expose, so the internal data can be used with MemoryCacherReadBlocks (see below) + UInt8** GetCacheBlocks () { return m_Blocks.empty () ? NULL : &*m_Blocks.begin (); } +}; + +#if SUPPORT_SERIALIZE_WRITE + +/// Used by SerializedFile to write to disk. +/// Currently it doesn't allow any seeking that is, you can only write blocks in consecutive order +class FileCacherWrite : public CacheWriterBase +{ +public: + FileCacherWrite (); + void InitWriteFile (const std::string& pathName, size_t cacheSize); + + virtual ~FileCacherWrite (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int block); + + virtual bool WriteHeaderAndCloseFile (void* data, size_t position, size_t size); + + virtual bool CompleteWriting (size_t size); + + virtual size_t GetCacheSize () const { return m_CacheSize; } + virtual std::string GetPathName() const; + virtual size_t GetFileLength () const { AssertString("Dont use"); return 0; } + +private: + void WriteBlock (int block); + + int m_Block; + UInt8* m_DataCache; + size_t m_CacheSize; + + File m_File; + bool m_Success; + bool m_Locked; + std::string m_Path; +}; + +#endif // SUPPORT_SERIALIZE_WRITE + + +/// Used by SerializedFile to read from disk +class FileCacherRead : public CacheReaderBase +{ +public: + FileCacherRead (const std::string& pathName, size_t cacheSize, size_t cacheCount); + ~FileCacherRead (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int block); + + virtual void DirectRead (void* data, size_t position, size_t size); + + virtual size_t GetFileLength () const { return m_FileSize; } + virtual size_t GetCacheSize () const { return m_CacheSize; } + virtual std::string GetPathName() const; + +private: + + // Finishes all reading, deletes all caches + void Flush (); + + struct CacheBlock + { + UInt8* data; + int block; + int lockCount; + unsigned timeStamp; + }; + + // Reads the cacheBlock from disk. + void ReadCacheBlock (CacheBlock& cacheBlock); + + /// Allocates an cache block at block + CacheBlock& AllocateCacheBlock (int block); + + // Frees the cache block with the smallest timestamp that is not locked. + // Returns whether or not a cache block could be freed. + bool FreeSingleCache (); + + typedef UNITY_MAP(kMemFile, int, CacheBlock) CacheBlocks; + CacheBlocks m_CacheBlocks; + + size_t m_CacheSize; + size_t m_MaxCacheCount; + size_t m_FileSize; + UInt32 m_TimeStamp; + std::string m_Path; + File m_File; + ProfilerAllocationHeader* m_RootHeader; + friend struct OpenFilesCache; + + #if DEBUG_LINEAR_FILE_ACCESS + int m_LastFileAccessPosition; + #endif + +}; + +enum +{ + // Evil: Must match the in CompressedFileStreamMemory.h, UncompressedFileStreamMemory.h" + kCacheBlockSize = 1024 * 100 +}; + +/// Used by SerializedFile to read from disk +class MemoryCacherReadBlocks : public CacheReaderBase +{ +public: + + MemoryCacherReadBlocks (UInt8** blocks, int size, size_t cacheBlockSize); + ~MemoryCacherReadBlocks (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int /*block*/) {} + + virtual void DirectRead (void* data, size_t position, size_t size); + + virtual size_t GetFileLength () const { return m_FileSize; } + + virtual size_t GetCacheSize () const { return m_CacheBlockSize; } + + virtual std::string GetPathName() const { return "none"; } + + virtual UInt8* GetAddressOfMemory() + { + return m_Memory[0]; + } + +private: + + // Finishes all reading, deletes all caches + void Flush (); + + UInt8** m_Memory; + size_t m_FileSize; + size_t m_CacheBlockSize; +}; + +void ReadFileCache (CacheReaderBase& cacher, void* data, size_t position, size_t size); +#if UNITY_EDITOR +void ForceCloseAllOpenFileCaches (); +#endif + +#endif + diff --git a/Runtime/Serialize/FloatStringConversion.cpp b/Runtime/Serialize/FloatStringConversion.cpp new file mode 100644 index 0000000..ce057d3 --- /dev/null +++ b/Runtime/Serialize/FloatStringConversion.cpp @@ -0,0 +1,80 @@ +#include "UnityPrefix.h" + +#if UNITY_EDITOR +// This are the definitions of std::numeric_limits<>::max_digits10, which we cannot use +// because it is only in the C++11 standard. +const int kMaxFloatDigits = std::floor(std::numeric_limits::digits * 3010.0/10000.0 + 2); +const int kMaxDoubleDigits = std::floor(std::numeric_limits::digits * 3010.0/10000.0 + 2); + +#include "External/gdtoa/gdtoa.h" + +bool FloatToStringAccurate (float f, char* buffer, size_t maximumSize) +{ + return g_ffmt (buffer, (float*)&f, kMaxFloatDigits, maximumSize) != NULL; +} + +bool DoubleToStringAccurate (double f, char* buffer, size_t maximumSize) +{ + return g_dfmt (buffer, (double*)&f, kMaxDoubleDigits, maximumSize) != NULL; +} + +bool FloatToStringAccurate (float f, UnityStr& output) +{ + char buf[64]; + if (FloatToStringAccurate (f, buf, 64)) + { + output = buf; + return true; + } + else + return false; +} + +bool DoubleToStringAccurate (double f, UnityStr& output) +{ + char buf[64]; + if (DoubleToStringAccurate (f, buf, 64)) + { + output = buf; + return true; + } + else + return false; +} + +float StringToFloatAccurate (const char* buffer) +{ + return strtof (buffer, NULL); +} + +double StringToDoubleAccurate (const char* buffer) +{ + return strtod (buffer, NULL); +} + +#if ENABLE_UNIT_TESTS +#include "../../External/UnitTest++/src/UnitTest++.h" + +SUITE (FloatStringConversionTests) +{ +TEST(FloatToStringConversion_AccurateWorks) +{ + // Make sure no locale is used + CHECK_EQUAL (0.0F, StringToFloatAccurate("0,5")); + + UnityStr buf; + + FloatToStringAccurate(1.0F, buf); + CHECK_EQUAL ("1", buf); + + CHECK_EQUAL (1.0F, StringToFloatAccurate("1.0")); + + + FloatToStringAccurate(1.5F, buf); + CHECK_EQUAL ("1.5", buf); + CHECK_EQUAL (1.5F, StringToFloatAccurate("1.5")); +} +} +#endif + +#endif diff --git a/Runtime/Serialize/FloatStringConversion.h b/Runtime/Serialize/FloatStringConversion.h new file mode 100644 index 0000000..c93d143 --- /dev/null +++ b/Runtime/Serialize/FloatStringConversion.h @@ -0,0 +1,16 @@ +#if UNITY_EDITOR + +/// Converts float/double to and from strings. + +/// Binary exact float<-> string conversion functions. +/// Supports the full range of all float values, including nan, inf and denormalized values. + +bool FloatToStringAccurate (float f, char* buffer, size_t maximumSize); +bool DoubleToStringAccurate (double f, char* buffer, size_t maximumSize); + +bool FloatToStringAccurate (float f, UnityStr& output); +bool DoubleToStringAccurate (double f, UnityStr& output); + +float StringToFloatAccurate (const char* buffer); +double StringToDoubleAccurate (const char* buffer); +#endif \ No newline at end of file diff --git a/Runtime/Serialize/IterateTypeTree.h b/Runtime/Serialize/IterateTypeTree.h new file mode 100644 index 0000000..4532b9b --- /dev/null +++ b/Runtime/Serialize/IterateTypeTree.h @@ -0,0 +1,155 @@ +#ifndef ITERATETYPETREE_H +#define ITERATETYPETREE_H + +#include "TypeTree.h" +#include "SerializeTraits.h" +#include "TransferUtility.h" + + +/* Iterate typetree is used to process serialized data in arbitrary ways. + +struct IterateTypeTreeFunctor +{ + // return true if you want to recurse into the function + bool operator () (const TypeTree& typeTree, dynamic_array& data, int bytePosition) + { + + } +} + +TypeTree typeTree; +dynamic_array data +// Create typetree and data +GenerateTypeTree(object); +WriteObjectToVector(object, &data); + +// Modify data +IterateTypeTreeFunctor func; +IterateTypeTree (typeTree, data, func); + +ReadObjectFromVector(&object, data, typeTree); +object.CheckConsistency (); +object.AwakeFromLoad (false); +object.SetDirty (); + +*/ + +inline SInt32 ExtractPPtrInstanceID (const UInt8* data) +{ + return *reinterpret_cast (data); +} + +inline SInt32 ExtractPPtrInstanceID (const dynamic_array& data, int bytePosition) +{ + return ExtractPPtrInstanceID(&data[bytePosition]); +} + +inline void SetPPtrInstanceID (SInt32 instanceID, dynamic_array& data, int bytePosition) +{ + *reinterpret_cast (&data[bytePosition]) = instanceID; +} + +inline bool IsTypeTreePPtr (const TypeTree& typeTree) +{ + return typeTree.m_Type.find ("PPtr<") == 0; +} + +inline bool IsTypeTreeString (const TypeTree& typeTree) +{ + return typeTree.m_Type == "string" && typeTree.m_Children.size() == 1 && typeTree.m_Children.back().m_IsArray; +} + +inline bool IsTypeTreePPtrArray (const TypeTree& typeTree) +{ + return typeTree.m_IsArray && typeTree.m_Children.back().m_Type.find ("PPtr<") == 0; +} + +inline bool IsTypeTreeArraySize (const TypeTree& typeTree) +{ + return typeTree.m_Father != NULL && typeTree.m_Father->m_IsArray && &typeTree.m_Father->m_Children.front() == &typeTree; +} + +inline bool IsTypeTreeArrayElement (const TypeTree& typeTree) +{ + return typeTree.m_Father != NULL && typeTree.m_Father->m_IsArray && &typeTree.m_Father->m_Children.back() == &typeTree; +} + +inline bool IsTypeTreeArrayOrArrayContainer (const TypeTree& typeTree) +{ + return typeTree.m_IsArray || (typeTree.m_Children.size() == 1 && typeTree.m_Children.back().m_IsArray); +} + +inline bool IsTypeTreeArray (const TypeTree& typeTree) +{ + return typeTree.m_IsArray; +} + +inline int ExtractArraySize (const UInt8* data) +{ + return *reinterpret_cast (data); +} + +inline void SetArraySize (UInt8* data, SInt32 size) +{ + *reinterpret_cast (data) = size; +} + +inline int ExtractArraySize (const dynamic_array& data, int bytePosition) +{ + return *reinterpret_cast (&data[bytePosition]); +} + + +inline UInt32 Align4_Iterate (UInt32 size) +{ + UInt32 value = ((size + 3) >> 2) << 2; + return value; +} +#if UNITY_EDITOR + +template +void IterateTypeTree (const TypeTree& typeTree, dynamic_array& data, Functor& functor) +{ + int bytePosition = 0; + IterateTypeTree (typeTree, data, &bytePosition, functor); +} +template +void IterateTypeTree (const TypeTree& typeTree, dynamic_array& data, int* bytePosition, Functor& functor) +{ + if (functor (typeTree, data, *bytePosition)) + { + if (typeTree.m_IsArray) + { + // First child in an array is the size + // Second child is the homogenous type of the array + AssertIf (typeTree.m_Children.front ().m_Type != SerializeTraits::GetTypeString (NULL) || typeTree.m_Children.front ().m_Name != "size" || typeTree.m_Children.size () != 2); + + functor (typeTree.m_Children.front (), data, *bytePosition); + + SInt32 arraySize, i; + arraySize = *reinterpret_cast (&data[*bytePosition]); + *bytePosition += sizeof (arraySize); + + for (i=0;isecond; + + if (create) + { + m_PathToStreamID.insert (make_pair (lowerCasePathName, m_PathNames.size ())); + m_PathNames.push_back (pathname); + AddStream (); + return m_PathNames.size () - 1; + } + else + return -1; +} + +int PathNamePersistentManager::InsertFileIdentifierInternal (FileIdentifier file, bool create) +{ + return InsertPathNameInternal(file.pathName, create); +} + +FileIdentifier PathNamePersistentManager::PathIDToFileIdentifierInternal (int pathID) +{ + AssertIf (pathID < 0 || pathID >= m_PathNames.size ()); + FileIdentifier f; + f.pathName = m_PathNames[pathID]; + return f; +} + +string PathNamePersistentManager::PathIDToPathNameInternal (int pathID) +{ + AssertIf (pathID < 0 || pathID >= m_PathNames.size ()); + return m_PathNames[pathID]; +} + +void InitPathNamePersistentManager() +{ + UNITY_NEW_AS_ROOT( PathNamePersistentManager(0), kMemManager, "PathNameManager", ""); +} diff --git a/Runtime/Serialize/PathNamePersistentManager.h b/Runtime/Serialize/PathNamePersistentManager.h new file mode 100644 index 0000000..11c3767 --- /dev/null +++ b/Runtime/Serialize/PathNamePersistentManager.h @@ -0,0 +1,31 @@ +#ifndef PATHNAMEPERSISTENTMANAGER_H +#define PATHNAMEPERSISTENTMANAGER_H + +#include "PersistentManager.h" +#include "SerializedFile.h" + +class PathNamePersistentManager : public PersistentManager +{ + typedef map PathToStreamID; + PathToStreamID m_PathToStreamID; // Contains lower case pathnames + vector m_PathNames;// Contains pathnames as they were given + + public: + + PathNamePersistentManager (int options, int cacheCount = 2) + : PersistentManager (options, cacheCount) {} + + protected: + + virtual int InsertPathNameInternal (const std::string& pathname, bool create); + virtual int InsertFileIdentifierInternal (FileIdentifier file, bool create); + + virtual string PathIDToPathNameInternal (int pathID); + + virtual FileIdentifier PathIDToFileIdentifierInternal (int pathID); +}; + +void InitPathNamePersistentManager(); + + +#endif diff --git a/Runtime/Serialize/PersistentManager.cpp b/Runtime/Serialize/PersistentManager.cpp new file mode 100644 index 0000000..3f596f4 --- /dev/null +++ b/Runtime/Serialize/PersistentManager.cpp @@ -0,0 +1,2291 @@ +#include "UnityPrefix.h" +#include "PersistentManager.h" +#include "Runtime/BaseClasses/BaseObject.h" +#include "Runtime/Utilities/FileUtilities.h" +#include "SerializedFile.h" +#include "Runtime/Utilities/LogAssert.h" +#include "Remapper.h" +#include "Runtime/Utilities/Word.h" +#include "Runtime/Profiler/Profiler.h" +#include "Runtime/Utilities/File.h" +#include "AwakeFromLoadQueue.h" + +#define DEBUG_THREAD_LOAD 0 +#define DEBUG_THREAD_LOAD_LONG_ACTIVATE !UNITY_RELEASE +#define DEBUG_MAINTHREAD_LOADING 0 + + +#if DEBUG_THREAD_LOAD +#define printf_debug_thread printf_console +#else +#define printf_debug_thread +#endif + +#include "Runtime/Threads/ProfilerMutex.h" +#if ENABLE_PROFILER +#include "Runtime/Profiler/MemoryProfiler.h" +#endif +PROFILER_INFORMATION(gMakeObjectUnpersistentProfiler, "Loading.MakeObjectUnpersistent", kProfilerLoading) +PROFILER_INFORMATION(gMakeObjectPersistentProfiler, "Loading.MakeObjectUnpersistent", kProfilerLoading) +PROFILER_INFORMATION(gAwakeFromLoadManager, "Loading.AwakeFromLoad", kProfilerLoading) +PROFILER_INFORMATION(gIDRemappingProfiler, "Loading.IDRemapping", kProfilerLoading) +PROFILER_INFORMATION(gWriteFileProfiler, "Loading.WriteFile", kProfilerLoading) +PROFILER_INFORMATION(gFindInActivationQueueProfiler, "Loading.FindInThreadedActivationQueue", kProfilerLoading) +PROFILER_INFORMATION(gReadObjectProfiler, "Loading.ReadObject", kProfilerLoading) +PROFILER_INFORMATION(gLoadFileProfiler, "Loading.LoadFile", kProfilerLoading) +PROFILER_INFORMATION(gIsObjectAvailable, "Loading.IsObjectAvailaable", kProfilerLoading) +PROFILER_INFORMATION(gLoadStreamNameSpaceProfiler, "Loading.LoadFileHeaders", kProfilerLoading) +PROFILER_INFORMATION(gLoadLockPersistentManager, "Loading.LockPersistentManager", kProfilerLoading) +PROFILER_INFORMATION(gLoadFromActivationQueueStall, "Loading.LoadFromActivationQueue stalled [wait for loading operation to finish]", kProfilerLoading) +// @TODO: Write test for cross references between monobehaviours in prefab and in scene! + + +static PersistentManager* gPersistentManager = NULL; +static PersistentManager::InOrderDeleteCallbackFunction* gInOrderDeleteCallback = NULL; +static PersistentManager::SafeBinaryReadCallbackFunction* gSafeBinaryReadCallback = NULL; + +static const char* kSerializedFileArea = "SerializedFile"; +static const char* kRemapperAllocArea = "PersistentManager.Remapper"; + +#if UNITY_EDITOR || SUPPORT_RESOURCE_IMAGE_LOADING +static const char* kResourceImageExtensions[] = { "resG", "res", "resS" }; +#endif + +double GetTimeSinceStartup (); + +using namespace std; + +bool PersistentManager::InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + return m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier); +} + +int PersistentManager::SerializedObjectIdentifierToInstanceID (const SerializedObjectIdentifier& identifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + return m_Remapper->GetOrGenerateMemoryID (identifier); +} + + +LocalIdentifierInFileType PersistentManager::GetLocalFileID(SInt32 instanceID) +{ + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier (instanceID, identifier); + return identifier.localIdentifierInFile; +} + +SInt32 PersistentManager::GetInstanceIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = InsertPathNameInternal (path, true); + identifier.localIdentifierInFile = localIdentifierInFile; + return m_Remapper->GetOrGenerateMemoryID (identifier); +} + +int PersistentManager::GetClassIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = InsertPathNameInternal (path, true); + identifier.localIdentifierInFile = localIdentifierInFile; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return -1; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return -1; + + return stream->GetClassID (identifier.localIdentifierInFile); +} + +static void CleanupStream (StreamNameSpace& stream) +{ + SerializedFile* oldFile = stream.stream; + stream.stream = NULL; + + UNITY_DELETE (oldFile, kMemSerialization); +} + +int PersistentManager::GetSerializedClassID (int instanceID) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier)) + return -1; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return -1; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return -1; + + return stream->GetClassID (identifier.localIdentifierInFile); +} + +void PersistentManager::GetAllFileIDs (const string& pathName, vector* objects) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, true); + SerializedFile* stream = GetSerializedFileInternal (serializedFileIndex); + if (stream == NULL) + return; + + stream->GetAllFileIDs (objects); +} + +bool PersistentManager::RemoveObjectsFromPath (const std::string& pathName) +{ +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + ASSERT_RUNNING_ON_MAIN_THREAD + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SInt32 serializedFileIndex = InsertPathNameInternal (pathName, false); + if (serializedFileIndex == -1) + return false; + + vector temp; + m_Remapper->RemoveCompletePathID(serializedFileIndex, temp); + + return true; +} + +void PersistentManager::MakeObjectUnpersistent (int memoryID, UnpersistMode mode) +{ + PROFILER_AUTO_THREAD_SAFE(gMakeObjectUnpersistentProfiler, NULL); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + ASSERT_RUNNING_ON_MAIN_THREAD + + Object* o = Object::IDToPointer (memoryID); + if (o && !o->IsPersistent ()) + { +// #if DEBUGMODE && !UNITY_RELEASE +// AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, GetMainThreadID(), "PersistentManager.MakeObjectUnpersistent"); +// AssertIf (m_Remapper->GetPathID (memoryID) != -1); +// #endif + return; + } + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + if (mode == kDestroyFromFile) + DestroyFromFileInternal (memoryID); + + m_Remapper->Remove (memoryID); + + if (o) + o->SetIsPersistent (false); +} + +#if UNITY_EDITOR +void PersistentManager::MakeObjectPersistent (int heapID, const string& pathName) +{ + MakeObjectPersistentAtFileID (heapID, 0, pathName); +} + +void PersistentManager::MakeObjectPersistentAtFileID (int heapID, LocalIdentifierInFileType fileID, const string& pathName) +{ + MakeObjectsPersistent (&heapID, &fileID, 1, pathName); +} + +void PersistentManager::MakeObjectsPersistent (const int* heapIDs, LocalIdentifierInFileType* fileIDs, int size, const string& pathName, int options) +{ + PROFILER_AUTO_THREAD_SAFE(gMakeObjectPersistentProfiler, NULL); + + AssertIf(!m_AllowLoadingFromDisk); + AssertIf(!Thread::EqualsCurrentThreadID(GetMainThreadID())); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + AssertIf(pathName.empty()); + SInt32 globalNameSpace = InsertPathNameInternal (pathName, true); + StreamNameSpace* streamNameSpace = NULL; + + for (int i=0;iIsPersistent ()) + { + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier(heapID, identifier); + AssertIf (identifier.serializedFileIndex == -1); + + // Return if the file and serializedFileIndex is not going to change + if (globalNameSpace == identifier.serializedFileIndex) + { + if (fileID == 0 || identifier.localIdentifierInFile == fileID) + continue; + } + + MakeObjectUnpersistent (heapID, kDestroyFromFile); + } + } + + if (streamNameSpace == NULL) + streamNameSpace = &GetStreamNameSpaceInternal (globalNameSpace); + + // Allocate an fileID for this object in the File + if (fileID == 0) + { + fileID = streamNameSpace->highestID; + if (streamNameSpace->stream) + fileID = max (streamNameSpace->highestID, streamNameSpace->stream->GetHighestID ()); + fileID++; + } + streamNameSpace->highestID = max (streamNameSpace->highestID, fileID); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = globalNameSpace; + identifier.localIdentifierInFile = fileID; + m_Remapper->SetupRemapping (heapID, identifier); + fileIDs[i] = fileID; + + if (o) + { + + AssertIf (o->TestHideFlag (Object::kDontSave) && (options & kAllowDontSaveObjectsToBePersistent) == 0); + o->SetIsPersistent (true); + o->SetDirty (); + } + } +} +#endif + + +void PersistentManager::LocalSerializedObjectIdentifierToInstanceIDInternal (const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID) +{ + int activeNameSpace = m_ActiveNameSpace.top(); + LocalSerializedObjectIdentifierToInstanceIDInternal (activeNameSpace, localIdentifier, outInstanceID); +} + +void PersistentManager::LocalSerializedObjectIdentifierToInstanceIDInternal (int activeNameSpace, const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + + LocalIdentifierInFileType localIdentifierInFile = localIdentifier.localIdentifierInFile; + int localSerializedFileIndex = localIdentifier.localSerializedFileIndex; + + if (localIdentifierInFile == 0) + { + outInstanceID = 0; + return; + } + + AssertIf (localSerializedFileIndex == -1); + + int globalFileIndex; + if (localSerializedFileIndex == 0) + globalFileIndex = activeNameSpace; + else + { + AssertIf (m_Streams[activeNameSpace].stream == NULL); + + AssertIf(activeNameSpace >= m_LocalToGlobalNameSpace.size() || activeNameSpace < 0); + + IDRemap::iterator found = m_LocalToGlobalNameSpace[activeNameSpace].find (localSerializedFileIndex); + + if (found != m_LocalToGlobalNameSpace[activeNameSpace].end ()) + { + globalFileIndex = found->second; + } + else + { + AssertString ("illegal LocalPathID in persistentmanager"); + outInstanceID = 0; + return; + } + } + + SerializedObjectIdentifier globalIdentifier; + globalIdentifier.serializedFileIndex = globalFileIndex; + globalIdentifier.localIdentifierInFile = localIdentifierInFile; + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + ApplyInstanceIDRemap(globalIdentifier); + #endif + + outInstanceID = m_Remapper->GetOrGenerateMemoryID (globalIdentifier); +} + +#if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD +void PersistentManager::ApplyInstanceIDRemap(SerializedObjectIdentifier& id) +{ + InstanceIDRemap::iterator foundIDRemap = m_InstanceIDRemap.find(id); + if (foundIDRemap != m_InstanceIDRemap.end()) + id = foundIDRemap->second; +} +#endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + +LocalSerializedObjectIdentifier PersistentManager::GlobalToLocalSerializedFileIndexInternal (const SerializedObjectIdentifier& globalIdentifier) +{ + LocalIdentifierInFileType localIdentifierInFile = globalIdentifier.localIdentifierInFile; + int localSerializedFileIndex; + + // Remap globalPathID to localPathID + int activeNameSpace = m_ActiveNameSpace.top (); + + IDRemap& globalToLocalNameSpace = m_GlobalToLocalNameSpace[activeNameSpace]; + IDRemap& localToGlobalNameSpace = m_LocalToGlobalNameSpace[activeNameSpace]; + + IDRemap::iterator found = globalToLocalNameSpace.find (globalIdentifier.serializedFileIndex); + if (found == globalToLocalNameSpace.end ()) + { + SET_ALLOC_OWNER(NULL); + AssertIf (activeNameSpace >= m_Streams.size()); + AssertIf (m_Streams[activeNameSpace].stream == NULL); + SerializedFile& serialize = *m_Streams[activeNameSpace].stream; + + serialize.AddExternalRef (PathIDToFileIdentifierInternal (globalIdentifier.serializedFileIndex)); + + localSerializedFileIndex = serialize.GetExternalRefs ().size (); + globalToLocalNameSpace[globalIdentifier.serializedFileIndex] = localSerializedFileIndex; + localToGlobalNameSpace[localSerializedFileIndex] = globalIdentifier.serializedFileIndex; + } + else + localSerializedFileIndex = found->second; + + // Setup local identifier + LocalSerializedObjectIdentifier localIdentifier; + + localIdentifier.localSerializedFileIndex = localSerializedFileIndex; + localIdentifier.localIdentifierInFile = localIdentifierInFile; + + return localIdentifier; +} + +void PersistentManager::InstanceIDToLocalSerializedObjectIdentifierInternal (SInt32 instanceID, LocalSerializedObjectIdentifier& localIdentifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + + AssertIf (m_ActiveNameSpace.empty ()); + if (instanceID == 0) + { + localIdentifier.localSerializedFileIndex = 0; + localIdentifier.localIdentifierInFile = 0; + return; + } + + SerializedObjectIdentifier globalIdentifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier (instanceID, globalIdentifier)) + { + localIdentifier.localSerializedFileIndex = 0; + localIdentifier.localIdentifierInFile = 0; + return; + } + + localIdentifier = GlobalToLocalSerializedFileIndexInternal(globalIdentifier); +} + +bool PersistentManager::IsInstanceIDFromCurrentFileInternal (SInt32 instanceID) +{ + if (instanceID == 0) + return false; + + SerializedObjectIdentifier globalIdentifier; + + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier (instanceID, globalIdentifier)) + return false; + + int activeNameSpace = m_ActiveNameSpace.top (); + return globalIdentifier.serializedFileIndex == activeNameSpace; +} + +#if UNITY_EDITOR + +int PersistentManager::GetSerializedFileIndexFromPath (const std::string& path) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + return InsertPathNameInternal (path, true); +} + +bool PersistentManager::TestNeedWriteFile (const string& pathName, const std::set* cachedDirtyPathsHint) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = InsertPathNameInternal (pathName, false); + return TestNeedWriteFileInternal(serializedFileIndex, cachedDirtyPathsHint); +} + +bool PersistentManager::TestNeedWriteFile (int globalFileIndex, const std::set* cachedDirtyPathsHint) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + return TestNeedWriteFileInternal(globalFileIndex, cachedDirtyPathsHint); +} + +bool PersistentManager::TestNeedWriteFileInternal (int globalFileIndex, const std::set* cachedDirtyPathsHint) +{ + if (globalFileIndex == -1) + return false; + + SerializedFile* stream = m_Streams[globalFileIndex].stream; + + bool isFileDirty = stream != NULL && stream->IsFileDirty (); + + // Something was deleted from the file. Must write it! + if (isFileDirty) + return true; + + // Use Dirty path indices to quickly determine if a file needs writing + if (cachedDirtyPathsHint != NULL) + return cachedDirtyPathsHint->count (globalFileIndex); + + Object* o; + // Find out if file needs to write to disk + // - Get all loaded objects that have registered themselves for being at that file + set loadedWriteObjects; + m_Remapper->GetAllLoadedObjectsAtPath (globalFileIndex, &loadedWriteObjects); + for (set::iterator i=loadedWriteObjects.begin ();i != loadedWriteObjects.end ();i++) + { + o = Object::IDToPointer (*i); + if (o && o->IsPersistent () && o->IsPersistentDirty ()) + return true; + } + + return false; +} + +void PersistentManager::CleanupStreamAndNameSpaceMapping (unsigned serializedFileIndex) +{ + // Unload the file any way + // This saves memory - especially when reimporting lots of assets like when rebuilding the library + CleanupStream(m_Streams[serializedFileIndex]); + + m_GlobalToLocalNameSpace[serializedFileIndex].clear (); + m_LocalToGlobalNameSpace[serializedFileIndex].clear (); +} + +static bool InitTempWriteFile (FileCacherWrite& writer, const std::string& path, unsigned cacheSize) +{ + string tempWriteFileName = GenerateUniquePathSafe (path); + if (tempWriteFileName.empty()) + return false; + + writer.InitWriteFile(path, cacheSize); + + return true; +} + +int PersistentManager::WriteFile (const std::string& path, BuildTargetSelection target, int options) +{ + PROFILER_AUTO_THREAD_SAFE(gWriteFileProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex; + serializedFileIndex = InsertPathNameInternal(path, false); + if (serializedFileIndex == -1) + return kNoError; + + bool needsWrite = TestNeedWriteFile(serializedFileIndex); + + // Early out + if (!needsWrite) + { + // @TODO: THIS SHOULD NOT BE HACKED IN HERE. Make test coverage against increased leaking then remove this and call CleanupStream explicitly. + CleanupStreamAndNameSpaceMapping(serializedFileIndex); + return kNoError; + } + + set writeObjects; + if (options & kDontReadObjectsFromDiskBeforeWriting) + { + GetLoadedInstanceIDsAtPath (path, &writeObjects); + Assert(!writeObjects.empty()); + } + else + { + // Load all writeobjects into memory + // (dont use LoadFileCompletely, since that reads all objects + // even those that might have been changed in memory) + GetInstanceIDsAtPath (path, &writeObjects); + } + + vector writeData; + + for (set::iterator i=writeObjects.begin ();i != writeObjects.end ();i++) + { + SInt32 instanceID = *i; + + // Force load object from disk. + Object* o = dynamic_instanceID_cast (instanceID); + + if (o == NULL) + continue; + + #if UNITY_EDITOR + // Disable text serialization for terrain data. Just too much data, you'd never want to merge. + int cid = o->GetClassID(); + if (IsClassNonTextSerialized(cid)) + options &= ~kAllowTextSerialization; + #endif + + AssertIf (o != NULL && !o->IsPersistent ()); + + SerializedObjectIdentifier identifier; + m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier); + + Assert (identifier.serializedFileIndex == serializedFileIndex); + + DebugAssertIf (!o->IsPersistent ()); + DebugAssertIf (m_Remapper->GetSerializedFileIndex (instanceID) != serializedFileIndex); + DebugAssertIf (!m_Remapper->IsSetup (identifier)); + + writeData.push_back(WriteData (identifier.localIdentifierInFile, instanceID, BuildUsageTag())); + } + + sort(writeData.begin(), writeData.end()); + + int result = WriteFileInternal(path, serializedFileIndex, &writeData[0], writeData.size(), NULL, target, options); + if (result != kNoError && options & kAllowTextSerialization) + // Try binary serialization as a fallback. + result = WriteFileInternal(path, serializedFileIndex, &writeData[0], writeData.size(), NULL, target, options &~kAllowTextSerialization); + + return result; +} + +int PersistentManager::WriteFileInternal (const string& path, int serializedFileIndex, const WriteData* writeObjectData, int size, VerifyWriteObjectCallback* verifyCallback, BuildTargetSelection target, int options) +{ + //printf_console("Writing file %s\n", pathName.c_str()); + + // Create writing tools + CachedWriter writer; + + FileCacherWrite serializedFileWriter; + FileCacherWrite resourceImageWriters[kNbResourceImages]; + if (!InitTempWriteFile (serializedFileWriter, "Temp/tempFile", kCacheSize)) + return kFileCouldNotBeWritten; + writer.InitWrite(serializedFileWriter); + + if (options & kBuildResourceImage) + { + for (int i=0;iSetDebugPath(PathIDToPathNameInternal(serializedFileIndex)); + GetMemoryProfiler()->SetRootAllocationObjectName(tempSerialize, tempSerialize->GetDebugPath().c_str()); + #endif + + tempSerialize->InitializeWrite (writer, target, options); + m_Streams[serializedFileIndex].stream = tempSerialize; + + m_ActiveNameSpace.push (serializedFileIndex); + + bool writeSuccess = true; + // Write Objects in fileID order + for (int i=0;i (instanceID); + shouldUnloadImmediately = true; + } + + // Object can not be loaded, don't write it + if (o == NULL) + { + continue; + } + } + + if (verifyCallback != NULL && !verifyCallback (o, target.platform)) + writeSuccess = false; + + tempSerialize->WriteObject (*o, localIdentifierInFile, writeObjectData[i].buildUsage); + o->ClearPersistentDirty (); + + if (shouldUnloadImmediately) + UnloadObject(o); + } + + m_ActiveNameSpace.pop(); + + writeSuccess = writeSuccess && tempSerialize->FinishWriting() && !tempSerialize->HasErrors(); + + // Delete temp stream + if (m_Streams[serializedFileIndex].stream != tempSerialize) + { + writeSuccess = false; + UNITY_DELETE (tempSerialize, kMemSerialization); + tempSerialize = NULL; + } + + // Delete mappings + CleanupStreamAndNameSpaceMapping(serializedFileIndex); + + if (!writeSuccess) + { +// ErrorString ("Writing file: " + path + " failed. The temporary file " + serializedFileWriter.GetPathName() + " couldn't be written."); + return kFileCouldNotBeWritten; + } + + // Atomically move the serialized file into the target location + string actualNewPathName = RemapToAbsolutePath (path); + if (!MoveReplaceFile (serializedFileWriter.GetPathName(), actualNewPathName)) + { + ErrorString ("File " + path + " couldn't be written. Because moving " + serializedFileWriter.GetPathName() + " to " + actualNewPathName + " failed."); + return kFileCouldNotBeWritten; + } + SetFileFlags(actualNewPathName, kFileFlagTemporary, 0); + + + if (options & kBuildResourceImage) + { + // Move the resource images into the target location + for (int i=0;i 0) + { + if (!MoveReplaceFile (tempWriteFileName, targetPath)) + { + ErrorString ("File " + path + " couldn't be written. Because moving " + tempWriteFileName + " to " + actualNewPathName + " failed."); + return kFileCouldNotBeWritten; + } + SetFileFlags(targetPath, kFileFlagTemporary, 0); + } + } + } + + return kNoError; +} + +#endif + +string PersistentManager::GetPathName (SInt32 memoryID) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SInt32 serializedFileIndex = m_Remapper->GetSerializedFileIndex (memoryID); + if (serializedFileIndex == -1) + return string (); + else + return PathIDToPathNameInternal (serializedFileIndex); +} + +void PersistentManager::RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex (const ThreadedAwakeData& awake) +{ + // Register instance ID first and then unlock integration mutex so there is no chance of an object being loaded twice + AssertIf(!awake.completedThreadAwake); + if (awake.object != NULL) + { + Object::RegisterInstanceID(awake.object); + + m_IntegrationMutex.Unlock(); + + AwakeFromLoadMode mode = (AwakeFromLoadMode)(kDidLoadFromDisk | kDidLoadThreaded); + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject(*awake.object, awake.oldType, mode, awake.checkConsistency, gSafeBinaryReadCallback); +} + else + { + m_IntegrationMutex.Unlock(); + } +} + + +#if THREADED_LOADING +void PersistentManager::AllowIntegrationWithTimeoutAndWait () +{ + m_IntegrationMutex.Lock(); + m_AllowIntegrateThreadedObjectsWithTimeout = true; + m_IntegrationMutex.Unlock(); + + // Wait until the integration thread has integrated all assets + while (true) + { + m_IntegrationMutex.Lock(); + + if (m_ThreadedObjectActivationQueue.empty()) + { + m_IntegrationMutex.Unlock(); + break; + } + else + { + m_IntegrationMutex.Unlock(); + } + + Thread::Sleep(0.1F); + } + + m_IntegrationMutex.Lock(); + m_AllowIntegrateThreadedObjectsWithTimeout = false; + m_IntegrationMutex.Unlock(); +} +#else +void PersistentManager::AllowIntegrationWithTimeoutAndWait () +{ + IntegrateAllThreadedObjects(); +} +#endif + +#if DEBUG_THREAD_LOAD +int gDependencyCounter = 0; +int gDependencyCounterCost = 0; +int gDependencyCounterCostActivation = 0; +int gDependencyCounterCostNonActivation = 0; +int gDependencyCounterCostNotFound = 0; +#endif + +Object* PersistentManager::LoadFromActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + ASSERT_RUNNING_ON_MAIN_THREAD + + LOCK_MUTEX(m_IntegrationMutex, gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator imap = m_ThreadedObjectActivationMap.find(heapID); + + if (imap != m_ThreadedObjectActivationMap.end()) + { + ThreadedObjectActivationQueue::iterator i = imap->second; + if (!i->completedThreadAwake) + { + ErrorString("Internal thread activation error. Activating object that has not been fully thread loaded."); + m_IntegrationMutex.Unlock(); + + return NULL; + } + + ThreadedAwakeData data = *i; + m_ThreadedObjectActivationQueue.erase(i); + m_ThreadedObjectActivationMap.erase(imap); + #if DEBUG_THREAD_LOAD +// printf_debug_thread("Activating dependency %d / %d [%d]\n", ++gDependencyCounter, gDependencyCounterCost, thisCost); +// if (thisCost > 300) +// { +// printf_debug_thread(""); +// } + #endif + + RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex(data); + + return data.object; + } + + #if DEBUG_THREAD_LOAD +// if (thisCost > 300) +// { +// printf_debug_thread("Expensive load from activation queue and not found [%d]\n", thisCost); +// } + #endif + + m_IntegrationMutex.Unlock(); + + return NULL; +} + +// GetFromActivationQueue is called a lot, so this function must be very efficient. +// In one of the games called Harvest, m_ThreadedObjectActivationQueue contained 70000 items, and this function was called 5678 times +// so linear searching was ubber slow!!! +Object* PersistentManager::GetFromActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_IntegrationMutex,gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator item = m_ThreadedObjectActivationMap.find(heapID); + if (item != m_ThreadedObjectActivationMap.end()) + { + return item->second->object; + } + + AssertIf(m_OnDemandThreadLoadedObjects.count(heapID)); + + return NULL; +} + +bool PersistentManager::FindInActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_IntegrationMutex, gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator item = m_ThreadedObjectActivationMap.find(heapID); + if (item != m_ThreadedObjectActivationMap.end()) + { + return true; + } + + if (m_OnDemandThreadLoadedObjects.count(heapID)) + return true; + + return false; +} + +bool PersistentManager::HasThreadedObjectsToIntegrate () +{ + Mutex::AutoLock lock (m_IntegrationMutex); + return !m_ThreadedObjectActivationQueue.empty (); +} + +///@TODO: Prevent Object destruction during AwakeFromLoad!!! + +void PersistentManager::IntegrateThreadedObjects (float timeout) +{ + // Early out if there is nothing to be integrated. + if (!m_AllowIntegrateThreadedObjectsWithTimeout) + return; + + if (!m_IntegrationMutex.TryLock()) + { + //printf_debug_thread("\nINTEGRATION THREAD IS LOCKED\n"); + return; + } + + if (m_ThreadedObjectActivationQueue.empty() || !m_AllowIntegrateThreadedObjectsWithTimeout) + { + m_IntegrationMutex.Unlock(); + return; + } + + double startTime = GetTimeSinceStartup(); + #if DEBUG_THREAD_LOAD + int integratedObjects = 0; + #endif + + while (true) + { + ThreadedAwakeData awake = m_ThreadedObjectActivationQueue.front(); + + if (!awake.completedThreadAwake) + { + ErrorString("Stalling Integration because ThreadAwake for object is not completed"); + break; + } + + #if DEBUG_THREAD_LOAD + integratedObjects++; + #endif + m_ThreadedObjectActivationMap.erase(awake.instanceID); + m_ThreadedObjectActivationQueue.pop_front(); + + #if DEBUG_THREAD_LOAD_LONG_ACTIVATE + double time = GetTimeSinceStartup(); + #endif + + RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex(awake); + + #if DEBUG_THREAD_LOAD_LONG_ACTIVATE + time = GetTimeSinceStartup() - time; + time *= 1000.0F; + if (time > 30.0F) + { + printf_console("Long thread activation time (%d ms) for object %s (%s)\n", (int)time, awake.object->GetName(), awake.object->GetClassName().c_str()); + } + #endif + + if (!m_AllowIntegrateThreadedObjectsWithTimeout || !m_IntegrationMutex.TryLock()) + { + #if DEBUG_THREAD_LOAD + startTime = (GetTimeSinceStartup() - startTime) * 1000.0F; + printf_debug_thread("Integrate Threaded Objects ms: %f %d\n\n", (float)startTime, integratedObjects); + #endif + return; + } + + double delta = (GetTimeSinceStartup() - startTime); + if (m_ThreadedObjectActivationQueue.empty() || delta > timeout) + break; + } + + #if DEBUG_THREAD_LOAD + startTime = (GetTimeSinceStartup() - startTime) * 1000.0F; + printf_debug_thread("Integrate Threaded Objects ms: %f %d\n\n", (float)startTime, integratedObjects); + #endif + + m_IntegrationMutex.Unlock(); +} + +void PersistentManager::IntegrateAllThreadedObjects () +{ + AwakeFromLoadQueue awakeQueue (kMemTempAlloc); + + PrepareAllThreadedObjectsStep1 (awakeQueue); + awakeQueue.RegisterObjectInstanceIDs(); + IntegrateAllThreadedObjectsStep2 (awakeQueue); +} + +void PersistentManager::PrepareAllThreadedObjectsStep1 (AwakeFromLoadQueue& awakeQueue) +{ + AQUIRE_AUTOLOCK (m_IntegrationMutex, gLoadFromActivationQueueStall); + + // Add to AwakeFromLoadQueue - this will take care of ensuring sort order + awakeQueue.Reserve(m_ThreadedObjectActivationQueue.size()); + + ThreadedObjectActivationQueue::iterator end = m_ThreadedObjectActivationQueue.end(); + for (ThreadedObjectActivationQueue::iterator i=m_ThreadedObjectActivationQueue.begin();i != end;++i ) + { + ThreadedAwakeData& awake = *i; + + if (awake.object) + { + Assert (awake.completedThreadAwake); + awakeQueue.Add(*awake.object, awake.oldType, awake.checkConsistency); + } + } + m_ThreadedObjectActivationQueue.clear(); + m_ThreadedObjectActivationMap.clear(); +} + +void PersistentManager::IntegrateAllThreadedObjectsStep2 (AwakeFromLoadQueue& awakeQueue) +{ + Assert(m_ThreadedObjectActivationQueue.empty() && m_ThreadedObjectActivationMap.empty()); + + // Invoke AwakeFromLoadQueue in sorted order + AwakeFromLoadMode awakeMode = (AwakeFromLoadMode)(kDidLoadFromDisk | kDidLoadThreaded); + + awakeQueue.PersistentManagerAwakeFromLoad(awakeMode, gSafeBinaryReadCallback); + + Assert(m_ThreadedObjectActivationQueue.empty() && m_ThreadedObjectActivationMap.empty()); +} + + +bool PersistentManager::ReloadFromDisk (Object* obj) +{ + PROFILER_AUTO_THREAD_SAFE(gReadObjectProfiler, obj); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(obj->GetInstanceID(), identifier)) + { + ErrorString("Trying to reload asset from disk that is not stored on disk"); + return false; + } + + SerializedFile* stream = GetSerializedFileInternal(identifier.serializedFileIndex); + if (stream == NULL) + { + ErrorString("Trying to reload asset but can't find object on disk"); + return false; + } + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + stream->ReadObject (identifier.localIdentifierInFile, obj->GetInstanceID(), kCreateObjectDefault, true, &oldType, &didTypeTreeChange, &obj); + m_ActiveNameSpace.pop (); + + // Awake the object + if (obj) + { + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (*obj, oldType, kDidLoadFromDisk, didTypeTreeChange, gSafeBinaryReadCallback); + } + + return true; +} + +Object* PersistentManager::ReadObject (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gReadObjectProfiler, NULL); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + #if !UNITY_EDITOR + AssertIf(heapID < 0); + #endif + + #if DEBUG_MAINTHREAD_LOADING + double time = GetTimeSinceStartup (); + #endif + + ASSERT_RUNNING_ON_MAIN_THREAD + + LOCK_MUTEX(m_Mutex, gLoadLockPersistentManager); + + Object* o = LoadFromActivationQueue(heapID); + if (o != NULL) +{ + m_Mutex.Unlock(); + return o; + } + + // Find and load the right stream + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + { + m_Mutex.Unlock(); + return NULL; + } + +#if DEBUGMODE + AssertIf(!m_IsLoadingSceneFile && StrICmp(GetPathNameExtension(PathIDToPathNameInternal(identifier.serializedFileIndex)),"unity") == 0); +#endif + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + { + #if DEBUG_MAINTHREAD_LOADING + LogString(Format("--- Loading from main thread failed loading stream %f", (GetTimeSinceStartup () - time) * 1000.0F)); + #endif + + m_Mutex.Unlock(); + return NULL; +} + + AssertIf(Object::IDToPointer (heapID) != NULL); + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + o = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectDefault, true, &oldType, &didTypeTreeChange, &o); + m_ActiveNameSpace.pop (); + + // Awake the object + if (o) + { + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (*o, oldType, kDidLoadFromDisk, didTypeTreeChange, gSafeBinaryReadCallback); + } + +// printf_console ("Read %d %s (%s)\n", heapID, o ? o->GetName () : "", o ? o->GetClassName ().c_str() : ""); + + m_Mutex.Unlock(); + + #if DEBUG_MAINTHREAD_LOADING + if (o) + { + LogString(Format("--- Loading from main thread %s (%s) %f ms", o->GetName(), o->GetClassName().c_str(), (GetTimeSinceStartup () - time) * 1000.0F)); + } + else + { + LogString(Format("--- Loading from main thread (NULL) %f ms", (GetTimeSinceStartup () - time) * 1000.0F)); + } + #endif + + return o; +} + +void PersistentManager::SetupThreadActivationQueueObject (ThreadedAwakeData& awakeData, TypeTree* oldType, bool didTypeTreeChange) +{ + Object* obj = awakeData.object; + if (obj != NULL) +{ + awakeData.oldType = oldType; + awakeData.checkConsistency = didTypeTreeChange; + obj->AwakeFromLoadThreaded(); + } + awakeData.completedThreadAwake = true; +} + +ThreadedAwakeData* PersistentManager::CreateThreadActivationQueueEntry (SInt32 instanceID) +{ + DebugAssertIf(Object::IDToPointerThreadSafe(instanceID) != NULL); + + ThreadedAwakeData awake; + awake.instanceID = instanceID; + awake.checkConsistency = false; + awake.completedThreadAwake = false; + awake.oldType = NULL; + awake.object = NULL; + + ThreadedAwakeData* result; + m_IntegrationMutex.Lock(); + + m_ThreadedObjectActivationQueue.push_back(awake); + result = &m_ThreadedObjectActivationQueue.back(); + m_ThreadedObjectActivationMap[instanceID] = --m_ThreadedObjectActivationQueue.end(); + + m_IntegrationMutex.Unlock(); + DebugAssertIf(Object::IDToPointerThreadSafe(instanceID) != NULL); + return result; +} + +void PersistentManager::CheckInstanceIDsLoaded (SInt32* heapIDs, int size) +{ + m_IntegrationMutex.Lock(); + + set ids; + + // Search through threaded object activation queue and activate the object if necessary. + ThreadedObjectActivationQueue::iterator i, end; + end = m_ThreadedObjectActivationQueue.end(); + for (i=m_ThreadedObjectActivationQueue.begin();i != end;i++) + ids.insert(i->instanceID); + + for (int j=0;jInstanceIDToSerializedObjectIdentifier(heapID, identifier)) + return NULL; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return NULL; +#if DEBUGMODE + AssertIf(!m_IsLoadingSceneFile && PathIDToPathNameInternal(identifier.serializedFileIndex).find(".unity") != string::npos); +#endif + AssertIf(Object::IDToPointerThreadSafe (heapID) != NULL); + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry(heapID); + + // Inject into m_OnDemandThreadLoadedObjects so we can track which objects have been read during a LoadFileCompletelyThreaded or LoadThreadedObjects. + // This prevents loading objects twice + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.insert(heapID); + m_IntegrationMutex.Unlock(); + + awakeData->object = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectFromNonMainThread, !m_Remapper->IsSceneID (heapID), &oldType, &didTypeTreeChange, &awakeData->object); + + m_ActiveNameSpace.pop (); + + o = awakeData->object; + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + + return o; +} + +void PersistentManager::LoadObjectsThreaded (SInt32* heapIDs, int size, LoadProgress* loadProgress) +{ + if (size == 0) + return; + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + vector heapIDsCopy; + heapIDsCopy.assign(heapIDs, heapIDs + size); + + // Check which objects are already loaded all at once to lock object creation only once for a short amount of time + // Since we have locked persistentmanager already no objects can be loaded in the mean time + CheckInstanceIDsLoaded(&heapIDsCopy[0], size); + + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + + for (int i=0;iItemProcessed (); + continue; + } + + m_IntegrationMutex.Lock(); + if (m_OnDemandThreadLoadedObjects.count(heapID)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + m_IntegrationMutex.Unlock(); + continue; + } + m_IntegrationMutex.Unlock(); + + DebugAssertIf(Object::IDToPointerThreadSafe(heapID) != NULL); + //DebugAssertIf(FindInActivationQueue(heapID)); + + // Find and load the right stream + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry (heapID); + awakeData->object = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectFromNonMainThread, true, &oldType, &didTypeTreeChange, &awakeData->object); + if (loadProgress) + loadProgress->ItemProcessed (); + + AssertIf (m_Remapper->IsSceneID (heapID)); + + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + + m_ActiveNameSpace.pop (); +} + + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.clear(); + m_IntegrationMutex.Unlock(); + + #if DEBUGMODE + m_IsLoadingSceneFile = false; + #endif + } + +int PersistentManager::LoadFileCompletelyThreaded (const std::string& pathname, LocalIdentifierInFileType* fileIDs, SInt32* instanceIDs, int size, bool loadScene, LoadProgress* loadProgress) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadFromActivationQueueStall); + + // Find Stream + int pathID = InsertPathNameInternal (pathname, true); + SerializedFile* stream = GetSerializedFileInternal (pathID); + if (stream == NULL) + return kFileCouldNotBeRead; + + AssertIf(fileIDs != NULL && size == -1); + AssertIf(instanceIDs != NULL && size == -1); + + // Get all file IDs we want to load and generate instance ids + vector fileIDsVector; + vector instanceIDsVector; + if (size == -1) + { + stream->GetAllFileIDs (&fileIDsVector); + fileIDs = &fileIDsVector[0]; + size = fileIDsVector.size(); + if (loadProgress) + loadProgress->totalItems += size; + instanceIDsVector.resize(size); + instanceIDs = &instanceIDsVector[0]; + } + + // In the editor we can not use preallocate ranges since fileID's might be completely arbitrary ranges + if (loadScene && !UNITY_EDITOR) + { + LocalIdentifierInFileType highestFileID = 0; + for (int i=0;iPreallocateIDs(highestFileID, pathID); + + for (int i=0;iIsSetup(SerializedObjectIdentifier(pathID, fileID))); + instanceIDs[i] = m_Remapper->m_ActivePreallocatedIDBase + fileID * 2; + } + + #if DEBUGMODE + //SHOULDN"T BE NEEDED!!!!!! - TAKE THIS OUT DEBUG ONLY!!!! + CheckInstanceIDsLoaded(&instanceIDs[0], size); + for (int i=0;iGetOrGenerateMemoryID (SerializedObjectIdentifier(pathID, fileID)); + + if (heapID == 0) + { + AssertString ("Loading an object that was made unpersistent but wasn't destroyed before reloading it"); + } + instanceIDs[i] = heapID; + } + // - Figure out which ones are already loaded + CheckInstanceIDsLoaded(&instanceIDs[0], size); + + #if UNITY_EDITOR + // Ugly hack to prevent IsPersistent to be on for scene objects. + // Recursive serialization is to be blamed for this. When that is fixed this can be removed. + if (loadScene) + { + m_Remapper->m_LoadingSceneInstanceIDs.assign_clear_duplicates(instanceIDs, instanceIDs + size); + } + #endif + } + + // Load all objects + m_ActiveNameSpace.push (pathID); + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + + for (int i=0;iItemProcessed (); + continue; + } + + m_IntegrationMutex.Lock(); + if (m_OnDemandThreadLoadedObjects.count(heapID)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + m_IntegrationMutex.Unlock(); + continue; + } + m_IntegrationMutex.Unlock(); + + DebugAssertIf(Object::IDToPointerThreadSafe (heapID) != NULL); + + TypeTree* oldType; + bool didTypeTreeChange; + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry(heapID); + awakeData->object = NULL; + stream->ReadObject (fileID, heapID, kCreateObjectFromNonMainThread, !loadScene, &oldType, &didTypeTreeChange, &awakeData->object); + if (loadProgress) + loadProgress->ItemProcessed (); + + ////@TODO: NEED TO HANDLE DELETION OF OBJECTS WHEN THAT HAPPENS BETWEEN threaded loading and activation + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + } + + #if DEBUGMODE + m_IsLoadingSceneFile = false; + #endif + + m_ActiveNameSpace.pop (); + + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.clear(); + m_IntegrationMutex.Unlock(); + if (loadScene) + { + #if UNITY_EDITOR + for (int i=0;iRemove (instanceIDs[i]); + m_Remapper->m_LoadingSceneInstanceIDs.clear(); + #else + m_Remapper->ClearPreallocateIDs(); + #endif + + } + + return kNoError; +} + +int PersistentManager::LoadFileCompletely (const string& path) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int result = GetPersistentManager().LoadFileCompletelyThreaded(path, NULL, NULL, -1, false, (LoadProgress*)NULL); + GetPersistentManager().IntegrateAllThreadedObjects (); + + return result; +} + + +#if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD +void PersistentManager::RemapInstanceIDOnLoad (const std::string& srcPath, LocalIdentifierInFileType srcLocalFileID, const std::string& dstPath, LocalIdentifierInFileType dstLocalFileID) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier src; + SerializedObjectIdentifier dst; + + src.serializedFileIndex = InsertPathNameInternal (srcPath, true); + src.localIdentifierInFile = srcLocalFileID; + Assert(src.serializedFileIndex != -1); + + // Destination path needs to be inserted (in unity_web_old case) + dst.serializedFileIndex = InsertPathNameInternal (dstPath, true); + dst.localIdentifierInFile = dstLocalFileID; + Assert(dst.serializedFileIndex != -1); + + m_InstanceIDRemap[src] = dst; +} +#endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + + +#if UNITY_EDITOR + +void PersistentManager::SuggestFileIDToHeapIDs (const string& pathname, map& fileIDToHeapIDHint) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = InsertPathNameInternal (pathname, true); + + SerializedFile* stream = GetSerializedFileInternal (serializedFileIndex); + if (stream == NULL) + return; + + vector fileIDs; + stream->GetAllFileIDs (&fileIDs); + + for (vector::iterator i=fileIDs.begin ();i!=fileIDs.end ();++i) + { + LocalIdentifierInFileType fileID = *i; + SerializedObjectIdentifier identifier (serializedFileIndex, fileID); + + // fileIDToHeapIDHint is a hint which is used to keep the same fileID when entering/exiting playmode for example. + // It is only a hint and may not be fulfilled + if (!m_Remapper->IsSetup(identifier)) + { + if (fileIDToHeapIDHint.count(fileID)) + { + int suggestedHeapID = fileIDToHeapIDHint.find(fileID)->second; + if (Object::IDToPointer (suggestedHeapID) == NULL && !m_Remapper->IsHeapIDSetup(suggestedHeapID)) + { + m_Remapper->SetupRemapping(suggestedHeapID, identifier); + } + } + } + } +} +#endif + +void PersistentManager::GetLoadedInstanceIDsAtPath (const string& pathName, set* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, false); + if (serializedFileIndex != -1) + { + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllLoadedObjectsAtPath (serializedFileIndex, objects); + } +} + +void PersistentManager::GetPersistentInstanceIDsAtPath (const string& pathName, set* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int pathID = InsertPathNameInternal (pathName, true); + if (pathID == -1) + return; + + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllPersistentObjectsAtPath (pathID, objects); +} + +void PersistentManager::GetInstanceIDsAtPath (const string& pathName, set* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, true); + if (serializedFileIndex == -1) + return; + + SerializedFile* serialize = GetSerializedFileInternal (serializedFileIndex); + if (serialize) + { + // Get all objects in the file + vector fileIDs; + serialize->GetAllFileIDs (&fileIDs); + for (vector::iterator i=fileIDs.begin ();i!=fileIDs.end ();++i) + { + SerializedObjectIdentifier identifier (serializedFileIndex, *i); + SInt32 memoryID = m_Remapper->GetOrGenerateMemoryID (identifier); + + if (memoryID != 0) + objects->insert (memoryID); + } + } + + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllLoadedObjectsAtPath (serializedFileIndex, objects); +} + +void PersistentManager::GetInstanceIDsAtPath (const string& pathName, vector* objects) +{ + set temp; + GetInstanceIDsAtPath(pathName, &temp); + objects->assign(temp.begin(), temp.end()); +} + +int PersistentManager::CountInstanceIDsAtPath (const string& pathName) +{ + set objects; + GetInstanceIDsAtPath (pathName, &objects); + return objects.size (); +} + +bool PersistentManager::IsObjectAvailable (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gIsObjectAvailable, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + //////////////////////// DO WE WANT THIS!!!!!!!!!!!!!!!!!!!!!!!!!!! + if (FindInActivationQueue(heapID)) + return true; + + SerializedObjectIdentifier identifier; + + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + return false; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + // Stream can't be found + if (stream == NULL) + return false; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return false; + + // Check if the class can be produced + int classID = stream->GetClassID (identifier.localIdentifierInFile); + Object::RTTI* rtti = Object::ClassIDToRTTI (classID); + if (rtti && !rtti->isAbstract) + return true; + else + return false; +} + +bool PersistentManager::IsObjectAvailableDontCheckActualFile (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gIsObjectAvailable, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = m_Remapper->GetSerializedFileIndex (heapID); + if (serializedFileIndex == -1) + return false; + else + return true; +} + + +void PersistentManager::Lock() +{ + LOCK_MUTEX(m_Mutex, gLoadLockPersistentManager); +} + +void PersistentManager::Unlock() +{ + m_Mutex.Unlock(); +} + +void PersistentManager::DoneLoadingManagers() +{ + // We are done loading managers. Start instance IDs from a high constant value here, + // so new managers and built-in resources can be added without changed instanceIDs + // used by the content. + + //AssertIf(m_Remapper->GetHighestInUseHeapID() > 10000); + + if (m_Remapper->m_HighestMemoryID < 10000) + { + m_Remapper->m_HighestMemoryID = 10000; + } + + Object::DoneLoadingManagers(); +} + + +PersistentManager& GetPersistentManager () +{ + AssertIf (gPersistentManager == NULL); + return *gPersistentManager; +} + +PersistentManager* GetPersistentManagerPtr () +{ + return gPersistentManager; +} + +void CleanupPersistentManager() +{ + UNITY_DELETE( gPersistentManager, kMemManager); + gPersistentManager = NULL; +} + +PersistentManager::PersistentManager (int options, int cacheCount) +// list node size is base data type size + two node pointers. +: +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + m_ThreadedAwakeDataPool (false, "Remapper pool", sizeof (ThreadedAwakeData) + sizeof(void*)*2, 16 * 1024), + m_ThreadedAwakeDataPoolMap (false, "RemapperMap pool", sizeof (ThreadedObjectActivationQueue::iterator) + sizeof(void*)*5, 16 * 1024), + m_ThreadedObjectActivationQueue (m_ThreadedAwakeDataPool), + m_ThreadedObjectActivationMap (std::less(), m_ThreadedAwakeDataPoolMap), +#endif + m_AllowIntegrateThreadedObjectsWithTimeout (false) +{ + AssertIf (gPersistentManager); + gPersistentManager = this; + m_Options = options; + m_CacheCount = cacheCount; + m_Remapper = UNITY_NEW_AS_ROOT(Remapper (),kMemSerialization, kRemapperAllocArea, ""); + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + #if DEBUGMODE + m_PreventLoadingFromFile = -1; + m_AllowLoadingFromDisk = true; + #endif + + InitializeStdConverters (); +} + +PersistentManager::~PersistentManager () +{ + AssertIf(m_Mutex.IsLocked()); + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (!m_ActiveNameSpace.empty ()); + + for (StreamContainer::iterator i=m_Streams.begin ();i!=m_Streams.end ();++i) + CleanupStream(*i); + + UNITY_DELETE(m_Remapper, kMemSerialization); + CleanupStdConverters(); +} + +#if DEBUGMODE +void PersistentManager::SetDebugAssertLoadingFromFile (const std::string& path) +{ + if (path.empty()) + { + m_PreventLoadingFromFile = -1; + } + else + { + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + m_PreventLoadingFromFile = InsertPathNameInternal (path, false); + } +} +#endif + + +static bool IsPathBuiltinResourceFile (const std::string& pathName) +{ + return StrICmp(pathName, "library/unity default resources") == 0 || StrICmp(pathName, "library/unity_web_old") == 0 + || StrICmp(pathName, "library/unity editor resources") == 0; +} + +StreamNameSpace& PersistentManager::GetStreamNameSpaceInternal (int nameSpaceID) +{ +#if DEBUGMODE + AssertIf(m_PreventLoadingFromFile == nameSpaceID); +#endif + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + return nameSpace; + + #if SUPPORT_SERIALIZATION_FROM_DISK + + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + // Load Stream + string pathName = PathIDToPathNameInternal (nameSpaceID); + if (pathName.empty ()) + return nameSpace; + + // File not found + string absolutePath = RemapToAbsolutePath (pathName); + if (!IsFileCreated (absolutePath)) + { + #if !UNITY_EDITOR + AssertString("PersistentManager: Failed to open file at path: " + pathName); + #endif + return nameSpace; + } + + // Is Builtin resource file? + int options = 0; + if (IsPathBuiltinResourceFile(pathName)) + options |= kIsBuiltinResourcesFile; + + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile, kMemSerialization, kSerializedFileArea, ""); + SET_ALLOC_OWNER(nameSpace.stream); + #if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(pathName); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); + #endif + + // Resource image loading is only supported in the player! + ResourceImageGroup group; + #if SUPPORT_RESOURCE_IMAGE_LOADING + for (int i=0;iInitializeRead (RemapToAbsolutePath (pathName), group, kCacheSize, m_CacheCount, options)) + { + CleanupStream(nameSpace); + return nameSpace; + } + + PostLoadStreamNameSpace(nameSpace, nameSpaceID); +#endif + + return m_Streams[nameSpaceID]; +} + +#if SUPPORT_SERIALIZATION_FROM_DISK +bool PersistentManager::LoadCachedFile (const std::string& pathName, const std::string actualAbsolutePath) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + ErrorString ("Tryng to load a stream which is already loaded."); + + // File not found + if (!IsFileCreated (actualAbsolutePath)) + return false; + + ResourceImageGroup group; + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + #if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(actualAbsolutePath); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); + #endif + if (!nameSpace.stream->InitializeRead (actualAbsolutePath, group, kCacheSize, m_CacheCount, kSerializeGameRelease)) + { + CleanupStream(nameSpace); + return false; + } + + nameSpace.stream->SetIsCachedFileStream(true); + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.insert(pathName); + + return true; +} +#endif + +bool PersistentManager::LoadMemoryBlockStream (const std::string& pathName, UInt8** data, int offset, int end, const char* url) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + AssertIf (nameSpace.stream); + + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + +#if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(url?url:pathName); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); +#endif + + int options = kSerializeGameRelease; + + // In NaCl & Flash, default resources are not loaded from disk, + // but also from a web stream. + // Need to set the proper flag here, so they don't get unloaded + // in Garbage Collection. + if (IsPathBuiltinResourceFile(pathName)) + options |= kIsBuiltinResourcesFile; + + if (!nameSpace.stream->InitializeMemoryBlocks (RemapToAbsolutePath(pathName), data, end, offset, options)) + { + CleanupStream(nameSpace); + return false; + } + + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + m_MemoryLoadedOrCachedPathsMutex.Lock(); + m_MemoryLoadedOrCachedPaths.insert(pathName); + m_MemoryLoadedOrCachedPathsMutex.Unlock(); + + return true; +} + + +#if SUPPORT_SERIALIZATION_FROM_DISK +bool PersistentManager::LoadExternalStream (const std::string& pathName, const std::string& absolutePath, int flags, int readOffset) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + return false; + + // File not found + if (!IsFileCreated (absolutePath)) + return false; + + ResourceImageGroup group; + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + +#if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(absolutePath); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); +#endif + + if (!nameSpace.stream->InitializeRead (absolutePath, group, kCacheSize, m_CacheCount, flags, readOffset)) + { + CleanupStream(nameSpace); + return false; + } + + nameSpace.stream->SetIsCachedFileStream(true); + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + m_MemoryLoadedOrCachedPathsMutex.Lock(); + m_MemoryLoadedOrCachedPaths.insert(pathName); + m_MemoryLoadedOrCachedPathsMutex.Unlock(); + + return true; +} +#endif + +void PersistentManager::PostLoadStreamNameSpace (StreamNameSpace& nameSpace, int nameSpaceID) +{ + nameSpace.highestID = std::max (nameSpace.highestID, nameSpace.stream->GetHighestID ()); + SET_ALLOC_OWNER ( this ); + const dynamic_block_vector& externalRefs = nameSpace.stream->GetExternalRefs (); + // Read all local pathnames and generate global<->localnamespace mapping + for (unsigned int i=0;i!=externalRefs.size ();i++) + { + int serializedFileIndex = InsertFileIdentifierInternal (externalRefs[i], true); + m_GlobalToLocalNameSpace[nameSpaceID][serializedFileIndex] = i + 1; + m_LocalToGlobalNameSpace[nameSpaceID][i + 1] = serializedFileIndex; + } + + // Setup global to self namespace mapping + m_GlobalToLocalNameSpace[nameSpaceID][nameSpaceID] = 0; + m_LocalToGlobalNameSpace[nameSpaceID][0] = nameSpaceID; +} + +bool PersistentManager::IsFileEmpty (const string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + SerializedFile* serialize = GetSerializedFileInternal (InsertPathNameInternal (pathName, true)); + if (serialize == NULL) + return true; + else + return serialize->IsEmpty (); +} + +#if UNITY_EDITOR +bool PersistentManager::DeleteFile (const string& pathName, DeletionFlags flag) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, true); + if (globalNameSpace == -1) + return false; + + StreamNameSpace& stream = GetStreamNameSpaceInternal (globalNameSpace); + + set objectsInFile; + GetInstanceIDsAtPath (pathName, &objectsInFile); + + for (set::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + MakeObjectUnpersistent (*i, kDontDestroyFromFile); + + if (flag & kDeleteLoadedObjects) + { + for (set::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + { + Object* obj = Object::IDToPointer(*i); + UnloadObject(obj); + } + + #if DEBUGMODE + for (set::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + AssertIf (PPtr (*i)); + #endif + } + + #if DEBUGMODE + for (set::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + { + if (m_Remapper->GetSerializedFileIndex (*i) != -1) + { + Object* obj = PPtr(*i); + ErrorStringObject("NOT CORRECTLY UNLOADED!!!!", obj); + } + } + #endif + + if (stream.stream) + CleanupStream(stream); + + m_GlobalToLocalNameSpace[globalNameSpace].clear (); + m_LocalToGlobalNameSpace[globalNameSpace].clear (); + + string absolutePath = RemapToAbsolutePath (pathName); + if (IsFileCreated (absolutePath)) + { + if (!::DeleteFile (absolutePath)) + return false; + } + return true; +} + +#endif + +void PersistentManager::UnloadNonDirtyStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + // printf_console("------ Unloading non dirty stream\n"); + int dirtyStreams = 0; + int loadedStreams = 0; + int unloadedStreams = 0; + for (int i=0;iIsMemoryStream () || nameSpace.stream->IsCachedFileStream()) + { + loadedStreams++; + continue; + } + + bool unloadStream = true; + #if UNITY_EDITOR + unloadStream = !nameSpace.stream->IsFileDirty(); + #endif + + if (unloadStream) + { + unloadedStreams++; + CleanupStream(nameSpace); + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } + else + { + FileIdentifier identifier = PathIDToFileIdentifierInternal(i); + printf_console("Can't unload serialized file because it is dirty: %s\n", identifier.pathName.c_str()); + dirtyStreams++; + loadedStreams++; + } + } + + printf_console("Unloading %d Unused Serialized files (Serialized files now loaded: %d / Dirty serialized files: %d)\n", unloadedStreams, loadedStreams, dirtyStreams); +// printf_console("Streams that can't be unloaded Files %d\n", loadedStreams); +// printf_console("ID mapping count %d -- %d \n", m_Remapper->m_FileToHeapID.size(), m_Remapper->m_FileToHeapID.size() * 24); +// printf_console("RESURRRECT count %d -- %d \n", m_Remapper->m_ResurrectHeapID.size(), m_Remapper->m_ResurrectHeapID.size() * 24); +} + + +void PersistentManager::UnloadStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + for (int i=0;i debug; + m_Remapper->GetAllLoadedObjectsAtPath (i, &debug); + AssertIf (!debug.empty ()); + #endif +*/ + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } +} + +void PersistentManager::UnloadMemoryStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + for (int i=0;iIsMemoryStream () && !nameSpace.stream->IsCachedFileStream()) + continue; + + #if DEBUGMODE + set debug; + m_Remapper->GetAllLoadedObjectsAtPath (i, &debug); + AssertIf (!debug.empty ()); + #endif + + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.clear(); +} + +void PersistentManager::UnloadStream (const std::string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, false); + if (nameSpaceID == -1) + return; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + if (nameSpace.stream == NULL) + return; + + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[nameSpaceID].clear (); + m_LocalToGlobalNameSpace[nameSpaceID].clear (); + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.erase(pathName); +} + +bool PersistentManager::HasMemoryOrCachedSerializedFile (const std::string& path) +{ + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + return m_MemoryLoadedOrCachedPaths.count(path) == 1; +} + +void PersistentManager::ResetHighestFileIDAtPath (const string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, true); + if (globalNameSpace == -1) + return; + + AssertIf (m_Streams[globalNameSpace].stream != NULL); + m_Streams[globalNameSpace].highestID = 0; +} + +#if UNITY_EDITOR +// AwakeFromLoadQueue only supports this in the editor +void PersistentManager::RegisterSafeBinaryReadCallback (SafeBinaryReadCallbackFunction* callback) +{ + AssertIf (gSafeBinaryReadCallback); + gSafeBinaryReadCallback = callback; +} +#endif + +void PersistentManager::RegisterInOrderDeleteCallback (InOrderDeleteCallbackFunction* callback) +{ + AssertIf (gInOrderDeleteCallback); + gInOrderDeleteCallback = callback; +} + +SerializedFile* PersistentManager::GetSerializedFileInternal (const string& path) +{ + return GetSerializedFileInternal(InsertPathNameInternal (path, true)); +} + +SerializedFile* PersistentManager::GetSerializedFileInternal (int serializedFileIndex) +{ + if (serializedFileIndex == -1) + return NULL; + + StreamNameSpace& stream = GetStreamNameSpaceInternal (serializedFileIndex); + return stream.stream; +} + + +bool PersistentManager::IsStreamLoaded (const std::string& pathName) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, false); + if (globalNameSpace == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[globalNameSpace]; + return nameSpace.stream != NULL; +} + +void PersistentManager::AddStream () +{ + m_Streams.push_back (StreamNameSpace ()); + m_GlobalToLocalNameSpace.push_back (IDRemap ()); + m_LocalToGlobalNameSpace.push_back (IDRemap ()); +} + +void PersistentManager::DestroyFromFileInternal (int memoryID) +{ + SerializedObjectIdentifier identifier; + m_Remapper->InstanceIDToSerializedObjectIdentifier(memoryID, identifier); + SerializedFile* serialize = GetSerializedFileInternal (identifier.serializedFileIndex); + if (serialize) + serialize->DestroyObject (identifier.localIdentifierInFile); +} + +string PersistentManager::RemapToAbsolutePath (const string& path) +{ + UserPathRemap::iterator found = m_UserPathRemap.find(path); + if (found != m_UserPathRemap.end()) + return found->second; + + return PathToAbsolutePath(path); +} + +void PersistentManager::SetPathRemap (const string& path, const string& absoluteRemappedPath) +{ + if (!absoluteRemappedPath.empty()) + { + Assert (m_UserPathRemap.count(path) == 0); + m_UserPathRemap.insert(make_pair(path, absoluteRemappedPath)); + } + else + { + Assert (m_UserPathRemap.count(path) == 1); + m_UserPathRemap.erase(path); + } +} + +#if UNITY_EDITOR +bool PersistentManager::IsClassNonTextSerialized( int cid ) +{ + return m_NonTextSerializedClasses.find (cid) != m_NonTextSerializedClasses.end(); +} +#endif diff --git a/Runtime/Serialize/PersistentManager.h b/Runtime/Serialize/PersistentManager.h new file mode 100644 index 0000000..73f3f67 --- /dev/null +++ b/Runtime/Serialize/PersistentManager.h @@ -0,0 +1,473 @@ +#ifndef PERSISTENTMANAGER_H +#define PERSISTENTMANAGER_H + + +#define SUPPORT_INSTANCE_ID_REMAP_ON_LOAD (UNITY_EDITOR || WEBPLUG) + +class SerializedFile; +class Object; +class TypeTree; +struct FileIdentifier; + +#include +#include +#include +#include +#include +#include +#include "Runtime/Utilities/vector_map.h" +#include "Configuration/UnityConfigure.h" +#include "Runtime/Utilities/CStringHash.h" +#include "Runtime/Threads/Thread.h" +#include "Runtime/Threads/Mutex.h" +#include "WriteData.h" +#include "Runtime/Utilities/MemoryPool.h" +#include "LoadProgress.h" + +using std::map; +using std::set; +using std::vector; +using std::string; +using std::stack; + +class AwakeFromLoadQueue; +class Remapper; + +struct StreamNameSpace +{ + SerializedFile* stream; + LocalIdentifierInFileType highestID; + + StreamNameSpace () { stream = NULL; highestID = 0; } +}; + +enum +{ + kNoError = 0, + kFileCouldNotBeRead = 1, + kTypeTreeIsDifferent = 2, + kFileCouldNotBeWritten = 3 +}; + +enum UnpersistMode { + kDontDestroyFromFile = 0, + kDestroyFromFile = 1 +}; + + +struct ThreadedAwakeData +{ + SInt32 instanceID; + TypeTree* oldType; + Object* object; + bool checkConsistency; /// refactor to safeLoaded + + // Has the object been fully loaded with AwakeFromLoadThreaded. + // We have to make sure the Object* is available already, so that recursive PPtr's to each other from Mono can correctly be resolved. + // In this case, neither object has been fully created, but we can setup pointers between them already. + bool completedThreadAwake; +}; + +struct SerializedObjectIdentifier +{ + SInt32 serializedFileIndex; + LocalIdentifierInFileType localIdentifierInFile; + + SerializedObjectIdentifier (SInt32 inSerializedFileIndex, LocalIdentifierInFileType inLocalIdentifierInFile) + : serializedFileIndex (inSerializedFileIndex) + , localIdentifierInFile (inLocalIdentifierInFile) + { } + + SerializedObjectIdentifier () + : serializedFileIndex (0) + , localIdentifierInFile (0) + { } + + + friend bool operator < (const SerializedObjectIdentifier& lhs, const SerializedObjectIdentifier& rhs) + { + if (lhs.serializedFileIndex < rhs.serializedFileIndex) + return true; + else if (lhs.serializedFileIndex > rhs.serializedFileIndex) + return false; + else + return lhs.localIdentifierInFile < rhs.localIdentifierInFile; + } + + friend bool operator != (const SerializedObjectIdentifier& lhs, const SerializedObjectIdentifier& rhs) + { + return lhs.serializedFileIndex != rhs.serializedFileIndex || lhs.localIdentifierInFile != rhs.localIdentifierInFile; + } +}; + +struct LocalSerializedObjectIdentifier; + +// There are three types of ids. +// fileID, is an id that is local to a file. It ranges from [1 ... kNameSpaceSize] +// heapID, is an id that was allocated for an object that was not loaded from disk. [1 ... infinity] +// pathID, is an id to a file. Every file has a unique id. They are not recycled unless you delete the PersistentManager + +class PersistentManager +{ + protected: + enum + { +// Recursive serialization causes the deflated stream to be reset repeatedly - use bigger cache chunks to limit the impact on load times. +#if UNITY_ANDROID + kCacheSize = 1024 * 256 +#elif UNITY_XBOX360 + kCacheSize = 1024 * 32 +#elif UNITY_WINRT || UNITY_BB10 + kCacheSize = 1024 * 64 +#else + kCacheSize = 1024 * 7 +#endif + }; + + typedef std::pair IntPair; + typedef vector_map, STL_ALLOCATOR(kMemSerialization, IntPair) > IDRemap; + typedef UNITY_VECTOR(kMemSerialization,StreamNameSpace) StreamContainer; + + StreamContainer m_Streams; + UNITY_VECTOR(kMemSerialization,IDRemap) m_GlobalToLocalNameSpace; + UNITY_VECTOR(kMemSerialization,IDRemap) m_LocalToGlobalNameSpace; + Remapper* m_Remapper; + + typedef std::pair StringPair; + typedef vector_map UserPathRemap; + UserPathRemap m_UserPathRemap; + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + typedef vector_map,STL_ALLOCATOR(kMemSerialization,SerializedObjectIdentifier) > InstanceIDRemap; + InstanceIDRemap m_InstanceIDRemap; + #endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + #if UNITY_EDITOR + UNITY_SET(kMemSerialization,int) m_NonTextSerializedClasses; + #endif + + SInt32 m_CacheCount; + + stack > m_ActiveNameSpace; + int m_Options; + + #if DEBUGMODE + bool m_AllowLoadingFromDisk; + bool m_IsLoadingSceneFile; + int m_PreventLoadingFromFile; + #endif + + UNITY_SET(kMemSerialization, std::string) m_MemoryLoadedOrCachedPaths; + + Mutex m_Mutex; + Mutex m_IntegrationMutex; + + Mutex m_MemoryLoadedOrCachedPathsMutex; + + bool m_AllowIntegrateThreadedObjectsWithTimeout; // Mutex protected by m_IntegrationMutex + +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + MemoryPool m_ThreadedAwakeDataPool; + MemoryPool m_ThreadedAwakeDataPoolMap; + typedef std::list > ThreadedObjectActivationQueue; + typedef std::map, memory_pool_explicit > ThreadedObjectActivationMap; +#else + typedef std::list ThreadedObjectActivationQueue; + typedef std::map ThreadedObjectActivationMap; +#endif + ThreadedObjectActivationQueue m_ThreadedObjectActivationQueue; // protected by m_IntegrationMutex + ThreadedObjectActivationMap m_ThreadedObjectActivationMap; // protected by m_IntegrationMutex + UNITY_SET(kMemSerialization, SInt32) m_OnDemandThreadLoadedObjects; /// DONT USE POOL HERE NOT THREAD SAFE + + StreamNameSpace& GetStreamNameSpaceInternal (int nameSpaceID); + void DestroyFromFileInternal (int memoryID); + public: + + PersistentManager (int options, int cacheCount); + virtual ~PersistentManager (); + + /// Loads all objects in pathName + /// Returns kNoError, kFileCouldNotBeRead + int LoadFileCompletely (const string& pathname); + + #if UNITY_EDITOR + // Makes an object persistent and generates a unique fileID in pathName + // The object can now be referenced by other objects that write to disk + void MakeObjectPersistent (int heapID, const string& pathName); + // Makes an object persistent if fileID == 0 a new unique fileID in pathName will be generated + // The object can now be referenced by other objects that write to disk + // If the object is already persistent in another file or another fileID it will be destroyed from that file. + void MakeObjectPersistentAtFileID (int heapID, LocalIdentifierInFileType fileID, const string& pathName); + + /// Batch multiple heapID's and fileID's into one path name. + /// on return fileID's will contain the file id's that were generated (if fileIds[i] is non-zero that fileID will be used instead) + enum { kMakePersistentDontRequireToBeLoadedAndDontUnpersist = 1 << 0, kAllowDontSaveObjectsToBePersistent = 1 << 1 }; + void MakeObjectsPersistent (const int* heapIDs, LocalIdentifierInFileType* fileIDs, int size, const string& pathName, int options = 0); + #endif + + // Makes an object unpersistent + void MakeObjectUnpersistent (int memoryID, UnpersistMode unpersistMode); + + bool RemoveObjectsFromPath (const std::string& pathName); + + // Returns the pathname the referenced object is stored at, if its not persistent empty string is returned + string GetPathName (SInt32 memoryID); + // Returns the localFileID the referenced object has inside its file. + + bool InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier); + int SerializedObjectIdentifierToInstanceID (const SerializedObjectIdentifier& identifier); + + LocalIdentifierInFileType GetLocalFileID(SInt32 instanceID); + + // Generates or returns an instanceID from path and fileID which can then + // be used to load the object at that instanceID + ///@TODO: RENAME TO LocalIdentifierInFile + SInt32 GetInstanceIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile); + + // Returns classID from path and fileID. + int GetClassIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile); + + // Reads the object referenced by heapID, if there is no object with heapID, the object will first be produced. + // Returns the created and read object, or NULL if the object couldnt be found or was destroyed. + Object* ReadObject (int heapID); + + // Unloads all streams that are open. + // After UnloadStreams is called files may be safely replaced. + // May only be called if there are no dirty files open. + void UnloadStreams (); + void UnloadStream (const std::string& pathName); + + bool IsStreamLoaded (const std::string& pathName); + + #if UNITY_EDITOR + + typedef bool VerifyWriteObjectCallback (Object* verifyDeployment, BuildTargetPlatform target); + + // Writes all persistent objects in memory that are made peristent at pathname to the file + // And completes all write operation (including writing the header) + // Returns the error (kNoError) + // options: kSerializeGameRelease, kSwapEndianess, kBuildPlayerOnlySerializeBuildProperties + int WriteFile (const string& pathName, BuildTargetSelection target = BuildTargetSelection::NoTarget(), int options = 0); + + bool IsClassNonTextSerialized(int cid); + + int WriteFileInternal (const std::string& path, int serializedFileIndex, const WriteData* writeData, int size, VerifyWriteObjectCallback* verifyCallback, BuildTargetSelection target, int options); + + #if UNITY_EDITOR + bool TestNeedWriteFile (const std::string& pathName, const std::set* dirtyPaths = NULL); + bool TestNeedWriteFile (int pathID, const std::set* dirtyPaths = NULL); + #endif + + // Delete file deletes the file referenced by pathName + // Makes all loaded objects unpersistent + // deleteLoadedObjects & kDeleteLoadedObjects -> All objects on the disk will be attempted to be destroyed + // deleteLoadedObjects & kDontDeleteLoadedObjects -> Doesn't delete any loaded objects, but marks them unpersistent from the file. + enum DeletionFlags { kDontDeleteLoadedObjects = 0, kDeleteLoadedObjects = 1 << 0 }; + bool DeleteFile (const string& pathName, DeletionFlags flag); + + void AddNonTextSerializedClass (int classID) { m_NonTextSerializedClasses.insert (classID); } + #endif + + // On return: objects are the instanceIDs of all objects resident in the file referenced by pathName + typedef std::set ObjectIDs; + void GetInstanceIDsAtPath (const string& pathName, ObjectIDs* objects); + void GetInstanceIDsAtPath (const string& pathName, vector* objects); + + void GetLoadedInstanceIDsAtPath (const string& pathName, ObjectIDs* objects); + void GetPersistentInstanceIDsAtPath (const string& pathName, std::set* objects); + + int CountInstanceIDsAtPath (const string& pathName); + + void SetAllowIntegrateThreadedObjectsWithTimeout (bool value); + + bool IsFileEmpty (const string& pathName); + + // Finds out if the referenced object can be loaded. + // By looking for it on the disk. And checking if the classID can be produced. + bool IsObjectAvailable (int heapID); + bool IsObjectAvailableDontCheckActualFile (int heapID); + + void GetAllFileIDs (const string& pathName, vector* objects); + + // Finds the + int GetSerializedClassID (int instanceID); + + + + + // Resets the fileIDs. This can only be used if the file has just been deleted. + void ResetHighestFileIDAtPath (const string& pathName); + + // Computes the memoryID (object->GetInstanceID ()) from fileID + // fileID is relative to the file we are currently writing/reading from. + // It can only be called when reading/writing objects in order to + // convert ptrs from file space to global space + void LocalSerializedObjectIdentifierToInstanceIDInternal (const LocalSerializedObjectIdentifier& identifier, SInt32& memoryID); + void LocalSerializedObjectIdentifierToInstanceIDInternal (int activeNameSpace, const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID); + + // fileID from memory ID (object->GetInstanceID ()) + // It can only be called when reading/writing objects in order + // to convert ptrs from global space to file space + void InstanceIDToLocalSerializedObjectIdentifierInternal (SInt32 memoryID, LocalSerializedObjectIdentifier& identifier); + + // Translates globalIdentifier.serializedFileIndex from a global index into the local file index based on what file we are currently writing. + // It can only be called when reading/writing objects in order + // to convert ptrs from global space to file space + LocalSerializedObjectIdentifier GlobalToLocalSerializedFileIndexInternal (const SerializedObjectIdentifier& globalIdentifier); + + /// Is this instanceID mapped to the file we are currently writing, + /// in other words is the referenced instanceID read or written from the same file then this will return true. + bool IsInstanceIDFromCurrentFileInternal (SInt32 memoryID); + + #if UNITY_EDITOR + // Hints a fileID to heap id mapping. + // This i used to keep similar instanceID's when entering / exiting playmode + void SuggestFileIDToHeapIDs (const string& pathname, std::map& fileIDToHeapIDHint); + + int GetSerializedFileIndexFromPath (const std::string& path); + + #endif + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + void RemapInstanceIDOnLoad (const std::string& srcPath, LocalIdentifierInFileType srcLocalFileID, const std::string& dstPath, LocalIdentifierInFileType dstLocalFileID); + void ApplyInstanceIDRemap(SerializedObjectIdentifier& id); + #endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + /// NOTE: Returns an object that has not been completely initialized (Awake has not been called yet) + Object* ReadObjectThreaded (int heapID); + + /// Check if we have objects to integrate, ie. if calling IntegrateThreadedObjects will perform some useful work + bool HasThreadedObjectsToIntegrate (); + + /// Integrates all thread loaded objects into the world (Called from PlayerLoop) + void IntegrateThreadedObjects (float timeout); + + /// Called from outise the loading thread (non-main thread), allows IntegrateThreadedObjects to be called with time slicing + /// Stalls until all objects have been integrated + void AllowIntegrationWithTimeoutAndWait (); + + void IntegrateAllThreadedObjects (); + + + void PrepareAllThreadedObjectsStep1 (AwakeFromLoadQueue& awakeQueue); + void IntegrateAllThreadedObjectsStep2 (AwakeFromLoadQueue& awakeQueue/*, bool loadScene*/); + + + + /// Load the entire file from a different thread + int LoadFileCompletelyThreaded (const std::string& pathname, LocalIdentifierInFileType* fileIDs, SInt32* instanceIDs, int size, bool loadScene, LoadProgress* loadProgress); + + /// Loads a number of objects threaded + void LoadObjectsThreaded (SInt32* heapIDs, int size, LoadProgress* loadProgress); + + #if SUPPORT_THREADS + const Mutex& GetMutex () { return m_Mutex; } + Thread::ThreadID GetMainThreadID () { return Thread::mainThreadId; } + #endif + + #if DEBUGMODE + /// Allows you to assert on any implicit loading operations for a specific file + void SetDebugAssertLoadingFromFile (const std::string& path); + /// Allows you to assert on any implicit loading operations, because for example when unloading objects that is usually not desired. + void SetDebugAssertAllowLoadingAnything (bool allowLoading) { m_AllowLoadingFromDisk = allowLoading; } + + void SetIsLoadingSceneFile(bool inexplicit) { m_IsLoadingSceneFile = inexplicit; } + #endif + + void Lock(); + void Unlock(); + /// Loads the contents of the object from disk again + /// Performs serialization and calls AwakeFromLoad + bool ReloadFromDisk (Object* obj); + + /// Load a memory stream directly from memory + /// - You should call this function only on assets that have been writting using kSerializeGameReleaswe + bool LoadMemoryBlockStream (const std::string& pathName, UInt8** data, int offset, int end, const char* url = NULL); + + // Loads a file at actualAbsolutePath and pretends that it is actually at path. + /// - You should call this function only on assets that have been writting using kSerializeGameReleaswe + bool LoadCachedFile (const std::string& path, const std::string actualAbsolutePath); + + void UnloadMemoryStreams (); + + // A registered SafeBinaryReadCallbackFunction will be called when an objects old typetree is different from the new one + // and variables might have gone away, added, or changed + typedef void SafeBinaryReadCallbackFunction (Object& object, const TypeTree& oldTypeTree); + static void RegisterSafeBinaryReadCallback (SafeBinaryReadCallbackFunction* callback); + + // In order for DeleteFile to work you have to support a callback that deletes the objects referenced by instanceID + typedef void InOrderDeleteCallbackFunction (const set& objects, bool safeDestruction); + static void RegisterInOrderDeleteCallback (InOrderDeleteCallbackFunction* callback); + + /// Thread locking must be performed from outside using Lock/Unlock + SerializedFile* GetSerializedFileInternal (const string& path); + SerializedFile* GetSerializedFileInternal (int serializedFileIndex); + + void SetPathRemap (const string& path, const string& absoluteRemappedPath); + + // Non-Locking method to find out if a memorystream or cached file is set up. + bool HasMemoryOrCachedSerializedFile (const std::string& path); + + #if SUPPORT_SERIALIZATION_FROM_DISK + bool LoadExternalStream (const std::string& pathName, const std::string& absolutePath, int flags, int readOffset = 0); + #endif + + void UnloadNonDirtyStreams (); + + /// NOTE: Function postfixed Internal are not thread safe and you have to call PersistentManager.Lock / Unlock prior to calling them from outside persistentmanager + + + //// Subclasses have to override these methods which map from PathIDs to FileIdentifier + /// Maps a pathname/fileidentifier to a pathID. If the pathname is not yet known, you have to call AddStream (). + /// The pathIDs start at 0 and increment by 1 + virtual int InsertFileIdentifierInternal (FileIdentifier file, bool create) = 0; + + std::string RemapToAbsolutePath (const std::string& path); + + void DoneLoadingManagers(); + + protected: + + virtual int InsertPathNameInternal (const std::string& pathname, bool create) = 0; + + /// maps a pathID to a pathname/file guid/fileidentifier. + /// (pathID can be assumed to be allocated before with InsertPathName) + virtual string PathIDToPathNameInternal (int pathID) = 0; + virtual FileIdentifier PathIDToFileIdentifierInternal (int pathID) = 0; + + /// Adds a new empty stream. Used by subclasses inside InsertPathName when a new pathID has to be added + void AddStream (); + + private: + + void CleanupStreamAndNameSpaceMapping (unsigned pathID); + + void RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex (const ThreadedAwakeData& awake); + + ThreadedAwakeData* CreateThreadActivationQueueEntry (SInt32 instanceID); + void SetupThreadActivationQueueObject (ThreadedAwakeData& data, TypeTree* oldType, bool didTypeTreeChange); + + /// Goes through object activation queue and calls AwakeFromLoad if it has been serialized already but not AwakeFromLoad called. + Object* LoadFromActivationQueue (int heapID); + Object* GetFromActivationQueue (int heapID); + bool FindInActivationQueue (int heapID); + void CheckInstanceIDsLoaded (SInt32* heapIDs, int size); + + protected: + + void PostLoadStreamNameSpace (StreamNameSpace& nameSpace, int namespaceID); + +#if UNITY_EDITOR + bool TestNeedWriteFileInternal (int pathID, const std::set* cachedDirtyPathsHint); +#endif + + friend class Object; +}; + +PersistentManager& GetPersistentManager (); +PersistentManager* GetPersistentManagerPtr (); + +void CleanupPersistentManager(); + +#endif diff --git a/Runtime/Serialize/Remapper.h b/Runtime/Serialize/Remapper.h new file mode 100644 index 0000000..6e1debc --- /dev/null +++ b/Runtime/Serialize/Remapper.h @@ -0,0 +1,297 @@ +#ifndef REMAPPER_H +#define REMAPPER_H + +#include +#include "Runtime/Utilities/MemoryPool.h" +#include "Configuration/UnityConfigure.h" + +class Remapper +{ + public: +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + MemoryPool m_SerializedObjectIdentifierPool; + typedef std::map, memory_pool_explicit > > FileToHeapIDMap; + typedef std::map, memory_pool_explicit > > HeapIDToFileMap; +#else + typedef std::map > FileToHeapIDMap; + typedef std::map > HeapIDToFileMap; +#endif + + typedef FileToHeapIDMap::iterator FileToHeapIDIterator; + typedef HeapIDToFileMap::iterator HeapIDToFileIterator; + + FileToHeapIDMap m_FileToHeapID; + HeapIDToFileMap m_HeapIDToFile; + #if UNITY_EDITOR + vector_set m_LoadingSceneInstanceIDs; + #endif + + + + // Instance ID's are simply allocated in an increasing index + int m_HighestMemoryID; + + // When loading scenes we can fast path because objects are not kept persistent / unloaded / loaded again etc. + // So we just preallocate a bunch of id's and use those without going through a lot of table lookups. + int m_ActivePreallocatedIDBase; + int m_ActivePreallocatedIDEnd; + int m_ActivePreallocatedPathID; + + Remapper () +// map node contains 3 pointers (left, right, parent) + +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP +: m_SerializedObjectIdentifierPool( false, "Remapper pool", sizeof (SerializedObjectIdentifier) + sizeof(SInt32)*2 + sizeof(void*)*3, 16 * 1024) +, m_FileToHeapID (std::less (), m_SerializedObjectIdentifierPool) +, m_HeapIDToFile (std::less (), m_SerializedObjectIdentifierPool) +#endif + { m_HighestMemoryID = 0; m_ActivePreallocatedIDBase = 0; m_ActivePreallocatedIDEnd = 0; m_ActivePreallocatedPathID = -1; } + + + void PreallocateIDs (LocalIdentifierInFileType highestFileID, int pathID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf(pathID == -1); + m_HighestMemoryID += 2; + m_ActivePreallocatedIDBase = m_HighestMemoryID; + m_HighestMemoryID += highestFileID * 2; + m_ActivePreallocatedIDEnd = m_HighestMemoryID; + //printf_console("Preallocating %d .. %d\n", m_ActivePreallocatedIDBase, m_ActivePreallocatedIDEnd); + m_ActivePreallocatedPathID = pathID; + } + + void ClearPreallocateIDs () + { + AssertIf(m_ActivePreallocatedPathID == -1); + m_ActivePreallocatedIDBase = 0; + m_ActivePreallocatedIDEnd = 0; + m_ActivePreallocatedPathID = -1; + } + + void Remove (int memoryID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + + HeapIDToFileIterator i = m_HeapIDToFile.find (memoryID); + if (i == m_HeapIDToFile.end ()) + return; + + FileToHeapIDIterator j = m_FileToHeapID.find (i->second); + AssertIf (j == m_FileToHeapID.end ()); + SerializedObjectIdentifier bug = j->first; + + m_HeapIDToFile.erase (i); + m_FileToHeapID.erase (j); + AssertIf (m_FileToHeapID.find (bug) != m_FileToHeapID.end ()); + } + + void RemoveCompletePathID (int serializedFileIndex, vector& objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + + SerializedObjectIdentifier proxy; + proxy.serializedFileIndex = serializedFileIndex; + proxy.localIdentifierInFile = std::numeric_limits::min(); + + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits::max(); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + for (FileToHeapIDIterator i=begin;i != end;i++) + { + ErrorIf(i->first.serializedFileIndex != serializedFileIndex); + m_HeapIDToFile.erase (m_HeapIDToFile.find(i->second)); + objects.push_back(i->second); + } + m_FileToHeapID.erase(begin, end); + } + + bool IsSceneID (int memoryID) + { +#if UNITY_EDITOR + return m_LoadingSceneInstanceIDs.count(memoryID) != 0; +#endif + return IsPreallocatedID (memoryID); + } + + bool IsPreallocatedID (int memoryID) + { + return m_ActivePreallocatedPathID != -1 && memoryID >= m_ActivePreallocatedIDBase && memoryID <= m_ActivePreallocatedIDEnd; + } + + bool InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier) + { + if (IsPreallocatedID(instanceID)) + { + identifier.serializedFileIndex = m_ActivePreallocatedPathID; + identifier.localIdentifierInFile = (instanceID - m_ActivePreallocatedIDBase) / 2; + return true; + } + + HeapIDToFileIterator i = m_HeapIDToFile.find (instanceID); + if (i == m_HeapIDToFile.end ()) + { + identifier.serializedFileIndex = -1; + identifier.localIdentifierInFile = 0; + return false; + } + identifier = i->second; + + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + + return true; + } + + + int GetSerializedFileIndex (int memoryID) + { + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier (memoryID, identifier); + return identifier.serializedFileIndex; + } + + bool IsSetup (const SerializedObjectIdentifier& identifier) + { + return m_FileToHeapID.find (identifier) != m_FileToHeapID.end (); + } + + bool IsHeapIDSetup (int memoryID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + return m_HeapIDToFile.count(memoryID); + } + + int GetOrGenerateMemoryID (const SerializedObjectIdentifier& identifier) + { + if (identifier.serializedFileIndex == -1) + return 0; + + if (m_ActivePreallocatedPathID != -1 && m_ActivePreallocatedPathID == identifier.serializedFileIndex) + { + return identifier.localIdentifierInFile * 2 + m_ActivePreallocatedIDBase; + } + + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + std::pair inserted = m_FileToHeapID.insert (std::make_pair (identifier, 0)); + if (inserted.second) + { + int memoryID = 0; + + m_HighestMemoryID += 2; + memoryID = m_HighestMemoryID; + + inserted.first->second = memoryID; + + AssertIf (m_HeapIDToFile.find (memoryID) != m_HeapIDToFile.end ()); + m_HeapIDToFile.insert (std::make_pair (memoryID, identifier)); + + return memoryID; + } + else + return inserted.first->second; + } + + void SetupRemapping (int memoryID, const SerializedObjectIdentifier& identifier) + { + AssertIf(m_ActivePreallocatedPathID != -1); + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + + if (m_HeapIDToFile.find (memoryID) != m_HeapIDToFile.end ()) + { + m_FileToHeapID.erase(m_HeapIDToFile.find (memoryID)->second); + m_HeapIDToFile.erase(memoryID); + } + + if (m_FileToHeapID.find (identifier) != m_FileToHeapID.end ()) + { + m_HeapIDToFile.erase(m_FileToHeapID.find (identifier)->second); + m_FileToHeapID.erase(identifier); + } + + m_HeapIDToFile[memoryID] = identifier; + m_FileToHeapID[identifier] = memoryID; + +/* +// This code asserts more when something goes wrong but also in edge cases that are allowed. + SerializedObjectIdentifier id; + id.fileID = fileID; + id.pathID = pathID; + + HeapIDToFileIterator inserted; + inserted = m_HeapIDToFile.insert (std::make_pair (memoryID, id)).first; + AssertIf (inserted->second != id); + inserted->second = id; + + FileToHeapIDIterator inserted2; + #if DEBUGMODE + inserted2 = m_FileToHeapID.find (id); + AssertIf (inserted2 != m_FileToHeapID.end () && inserted2->second != memoryID); + #endif + + inserted2 = m_FileToHeapID.insert (std::make_pair (id, memoryID)).first; + AssertIf (inserted2->second != memoryID); + inserted2->second = memoryID; +*/ + } + + void GetAllLoadedObjectsAtPath (int pathID, set* objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf (objects == NULL); + + SerializedObjectIdentifier proxy; + proxy.localIdentifierInFile = std::numeric_limits::min (); + proxy.serializedFileIndex = pathID; + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits::max (); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + + for (FileToHeapIDIterator i=begin;i != end;++i) + { + int instanceID = i->second; + Object* o = Object::IDToPointer (instanceID); + if (o) + objects->insert (instanceID); + } + } + + void GetAllPersistentObjectsAtPath (int pathID, set* objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf (objects == NULL); + + SerializedObjectIdentifier proxy; + proxy.localIdentifierInFile = std::numeric_limits::min (); + proxy.serializedFileIndex = pathID; + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits::max (); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + + for (FileToHeapIDIterator i=begin;i != end;++i) + { + int instanceID = i->second; + objects->insert(instanceID); + } + } + + int GetHighestInUseHeapID () + { + if (!m_HeapIDToFile.empty()) + return m_HeapIDToFile.rbegin ()->first; + else + return 0; + } +}; + +#endif diff --git a/Runtime/Serialize/SerializationMetaFlags.h b/Runtime/Serialize/SerializationMetaFlags.h new file mode 100644 index 0000000..fc47207 --- /dev/null +++ b/Runtime/Serialize/SerializationMetaFlags.h @@ -0,0 +1,292 @@ +#ifndef SERIALIZATIONMETAFLAGS_H +#define SERIALIZATIONMETAFLAGS_H + +#include "Runtime/Utilities/EnumFlags.h" + +/// Meta flags can be used like this: +/// transfer.Transfer (someVar, "varname", kHideInEditorMask); +/// The proxytransfer for example reads the metaflag mask and stores it in the TypeTree +enum TransferMetaFlags +{ + kNoTransferFlags = 0, + /// Putting this mask in a transfer will make the variable be hidden in the property editor + kHideInEditorMask = 1 << 0, + + /// Makes a variable not editable in the property editor + kNotEditableMask = 1 << 4, + + /// There are 3 types of PPtrs: kStrongPPtrMask, default (weak pointer) + /// a Strong PPtr forces the referenced object to be cloned. + /// A Weak PPtr doesnt clone the referenced object, but if the referenced object is being cloned anyway (eg. If another (strong) pptr references this object) + /// this PPtr will be remapped to the cloned object + /// If an object referenced by a WeakPPtr is not cloned, it will stay the same when duplicating and cloning, but be NULLed when templating + kStrongPPtrMask = 1 << 6, + // unused = 1 << 7, + + /// kEditorDisplaysCheckBoxMask makes an integer variable appear as a checkbox in the editor + kEditorDisplaysCheckBoxMask = 1 << 8, + + // unused = 1 << 9, + // unused = 1 << 10, + + /// Show in simplified editor + kSimpleEditorMask = 1 << 11, + + /// When the options of a serializer tells you to serialize debug properties kSerializeDebugProperties + /// All debug properties have to be marked kDebugPropertyMask + /// Debug properties are shown in expert mode in the inspector but are not serialized normally + kDebugPropertyMask = 1 << 12, + + kAlignBytesFlag = 1 << 14, + kAnyChildUsesAlignBytesFlag = 1 << 15, + kIgnoreWithInspectorUndoMask= 1 << 16, + + // unused = 1 << 18, + + // Ignore this property when reading or writing .meta files + kIgnoreInMetaFiles = 1 << 19, + + // When reading meta files and this property is not present, read array entry name instead (for backwards compatibility). + kTransferAsArrayEntryNameInMetaFiles = 1 << 20, + + // When writing YAML Files, uses the flow mapping style (all properties in one line, with "{}"). + kTransferUsingFlowMappingStyle = 1 << 21, + + // Tells SerializedProperty to generate bitwise difference information for this field. + kGenerateBitwiseDifferences = 1 << 22, + + kDontAnimate = 1 << 23, + +}; +ENUM_FLAGS(TransferMetaFlags); + +enum TransferInstructionFlags +{ + kNoTransferInstructionFlags = 0, + + kNeedsInstanceIDRemapping = 1 << 0, // Should we convert PPtrs into pathID, fileID using the PerisistentManager or should we just store the memory InstanceID in the fileID? + kAssetMetaDataOnly = 1 << 1, // Only serialize data needed for .meta files + kYamlGlobalPPtrReference = 1 << 2, + #if UNITY_EDITOR + kLoadAndUnloadAssetsDuringBuild = 1 << 3, + kSerializeDebugProperties = 1 << 4, // Should we serialize debug properties (eg. Serialize mono private variables) + #endif + kIgnoreDebugPropertiesForIndex = 1 << 5, // Should we ignore Debug properties when calculating the TypeTree index + #if UNITY_EDITOR + kBuildPlayerOnlySerializeBuildProperties = 1 << 6, // Used by eg. build player to make materials cull any properties are aren't used anymore ! + #endif + kWorkaround35MeshSerializationFuckup = 1 << 7, + + kSerializeGameRelease = 1 << 8, // Should Transfer classes use optimized reading. Allowing them to read memory directly that normally has a type using ReadDirect. + kSwapEndianess = 1 << 9, // Should we swap endianess when reading / writing a file + kSaveGlobalManagers = 1 << 10, // Should global managers be saved when writing the game build + kDontReadObjectsFromDiskBeforeWriting = 1 << 11, + kSerializeMonoReload = 1 << 12, // Should we backupmono mono variables for an assembly reload? + kDontRequireAllMetaFlags = 1 << 13, // Can we fast path calculating all meta data. This lets us skip a bunch of code when serializing mono data. + kSerializeForPrefabSystem = 1 << 14, + #if UNITY_EDITOR + kWarnAboutLeakedObjects = 1 << 15, + // Unused = 1 << 16, + // Unused = 1 << 17, + kEditorPlayMode = 1 << 18, + kBuildResourceImage = 1 << 19, + kSerializeEditorMinimalScene = 1 << 21, + kGenerateBakedPhysixMeshes = 1 << 22, + #endif + kThreadedSerialization = 1 << 23, + kIsBuiltinResourcesFile = 1 << 24, + kPerformUnloadDependencyTracking = 1 << 25, + kDisableWriteTypeTree = 1 << 26, + kAutoreplaceEditorWindow = 1 << 27,// Editor only + kSerializeForInspector = 1 << 29, + kSerializedAssetBundleVersion = 1 << 30, // When writing (typetrees disabled), allow later Unity versions an attempt to read SerializedFile. + kAllowTextSerialization = 1 << 31 +}; +ENUM_FLAGS(TransferInstructionFlags); + +enum BuildAssetBundleOptions +{ + kAssetBundleUncompressed = 1 << 11, + kAssetBundleCollectDependencies = 1 << 20, + kAssetBundleIncludeCompleteAssets = 1 << 21, + kAssetBundleDisableWriteTypeTree = 1 << 26, + kAssetBundleDeterministic = 1 << 28, +}; +ENUM_FLAGS(BuildAssetBundleOptions); + + +enum ActiveResourceImage +{ + kResourceImageNotSupported = -2, + kResourceImageInactive = -1, + kGPUResourceImage = 0, + kResourceImage = 1, + kStreamingResourceImage = 2, + kNbResourceImages = 3 +}; + +/// This needs to be in Sync with BuildTarget in C# +enum BuildTargetPlatform +{ + kBuildNoTargetPlatform = -2, + kBuildAnyPlayerData = -1, + kBuildValidPlayer = 1, + + // We don't support building for these any more, but we still need the constants for asset bundle + // backwards compatibility. + kBuildStandaloneOSXPPC = 3, + + kBuildStandaloneOSXIntel = 4, + kBuildStandaloneOSXIntel64 = 27, + kBuildStandaloneOSXUniversal = 2, + kBuildStandaloneWinPlayer = 5, + kBuildWebPlayerLZMA = 6, + kBuildWebPlayerLZMAStreamed = 7, + kBuildWii = 8, + kBuild_iPhone = 9, + kBuildPS3 = 10, + kBuildXBOX360 = 11, + // was kBuild_Broadcom = 12, + kBuild_Android = 13, + kBuildWinGLESEmu = 14, + // was kBuildWinGLES20Emu = 15, + kBuildNaCl = 16, + kBuildStandaloneLinux = 17, + kBuildFlash = 18, + kBuildStandaloneWin64Player = 19, + kBuildWebGL = 20, + kBuildMetroPlayerX86 = 21, + kBuildMetroPlayerX64 = 22, + kBuildMetroPlayerARM = 23, + kBuildStandaloneLinux64 = 24, + kBuildStandaloneLinuxUniversal = 25, + kBuildWP8Player = 26, + kBuildBB10 = 28, + kBuildTizen = 29, + kBuildPlayerTypeCount = 30, +}; + +struct BuildUsageTag +{ + bool forceTextureReadable; + bool strippedPrefabObject; + UInt32 meshUsageFlags; + UInt32 meshSupportedChannels; + + BuildUsageTag () + { + forceTextureReadable = false; + meshUsageFlags = 0; + meshSupportedChannels = 0; + strippedPrefabObject = false; + } +}; + + +struct BuildTargetSelection +{ + BuildTargetPlatform platform; + int subTarget; + + BuildTargetSelection() : platform(kBuildNoTargetPlatform), subTarget(0) { } + BuildTargetSelection(BuildTargetPlatform platform_, int subTarget_) : platform(platform_), subTarget(subTarget_) {} + + bool operator == (const BuildTargetSelection& rhs) const + { + if (platform != rhs.platform) + return false; + if (subTarget != rhs.subTarget) + return false; + + return true; + } + bool operator != (const BuildTargetSelection& rhs) const + { + return ! operator == (rhs); + } + + static BuildTargetSelection NoTarget() { return BuildTargetSelection(kBuildNoTargetPlatform,0); } +}; + + +enum WebPlayerBuildSubTarget +{ + kWebBuildSubtargetDefault = 0, + kWebBuildSubtargetDirect3D = 1, // windows only (D3D9 & D3D11) + kWebBuildSubtargetOpenGL = 2, // non-windows only (OpenGL) +}; + + +/// This needs to be in Sync with XboxRunMethod in C# +enum XboxBuildSubtarget +{ + kXboxBuildSubtargetDevelopment = 0, + kXboxBuildSubtargetMaster = 1, + kXboxBuildSubtargetDebug = 2 +}; + +/// This needs to be in Sync with WiiBuildDebugLevel in C# +enum WiiBuildDebugLevel +{ + kWiiBuildDebugLevel_Full = 0, + kWiiBuildDebugLevel_Minimal = 1, + kWiiBuildDebugLevel_None = 2, +}; + +/// This needs to be in Sync with XboxRunMethod in C# +enum XboxRunMethod +{ + kXboxRunMethodHDD = 0, + kXboxRunMethodDiscEmuFast = 1, + kXboxRunMethodDiscEmuAccurate = 2 +}; + +/// This needs to be in Sync with AndroidBuildSubtarget in C# +enum AndroidBuildSubtarget +{ + kAndroidBuildSubtarget_Generic = 0, + kAndroidBuildSubtarget_DXT = 1, + kAndroidBuildSubtarget_PVRTC = 2, + kAndroidBuildSubtarget_ATC = 3, + kAndroidBuildSubtarget_ETC = 4, + kAndroidBuildSubtarget_ETC2 = 5, + kAndroidBuildSubtarget_ASTC = 6, +}; + +/// This needs to be in Sync with BB10BuildSubtarget in C# +enum BlackBerryBuildSubtarget +{ + kBlackBerryBuildSubtarget_Generic = 0, + kBlackBerryBuildSubtarget_PVRTC = 1, + kBlackBerryBuildSubtarget_ATC = 2, + kBlackBerryBuildSubtarget_ETC = 3 +}; + +/// This needs to be in Sync with BB10BuildType in C# +enum BlackBerryBuildType +{ + kBlackBerryBuildType_Debug = 0, + kBlackBerryBuildType_Submission = 1 +}; + +/// This needs to be in Sync with BuildOptions in C# +enum BuildPlayerOptions +{ + kBuildPlayerOptionsNone = 0, + kDevelopmentBuild = 1 << 0, + kAutoRun = 1 << 2, + kSelectBuiltPlayer = 1 << 3, + kBuildAdditionalStreamedScenes = 1 << 4, + kAcceptExternalModificationsToPlayer = 1 << 5, + kInstallInBuildsFolder = 1 << 6, + kWebPlayerOfflineDeployment = 1 << 7, + kConnectWithProfiler = 1 << 8, + kAllowDebugging = 1 << 9, + kSymlinkLibraries = 1 << 10, + kBuildPlayerUncompressed = 1 << 11, + kConnectToHost = 1 << 12, + kDeployOnline = 1 << 13, + kHeadlessModeEnabled = 1 << 14 +}; + +#endif diff --git a/Runtime/Serialize/SerializationTests.cpp b/Runtime/Serialize/SerializationTests.cpp new file mode 100644 index 0000000..def94dd --- /dev/null +++ b/Runtime/Serialize/SerializationTests.cpp @@ -0,0 +1,242 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" + +#if ENABLE_UNIT_TESTS + +#include "Runtime/Testing/Testing.h" +#include "Runtime/Testing/TestFixtures.h" + +SUITE (SerializationTests) +{ + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidReadExistingProperty) + { + float m_FloatProperty; + TRANSFER (m_FloatProperty); + if (transfer.IsReading ()) + { + CHECK (transfer.DidReadLastProperty ()); + } + } + + TEST_FIXTURE (DidReadExistingPropertyTestFixture, SafeBinaryRead_DidReadLastProperty_WithExistingProperty_IsTrue) + { + DoSafeBinaryTransfer (); + } + + TEST_FIXTURE (DidReadExistingPropertyTestFixture, YAMLRead_DidReadLastProperty_WithExistingProperty_IsTrue) + { + DoTextTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidNotReadMissingProperty) + { + float m_Foobar; + TRANSFER (m_Foobar); + + if (transfer.IsReading ()) + { + UnityStr value = "foobar"; + TRANSFER (value); + + CHECK (!transfer.DidReadLastProperty ()); + CHECK (value == "foobar"); + } + } + + TEST_FIXTURE (DidNotReadMissingPropertyTestFixture, SafeBinaryRead_DidReadLastProperty_WithMissingProperty_IsFalse) + { + DoSafeBinaryTransfer (); + } + + TEST_FIXTURE (DidNotReadMissingPropertyTestFixture, YAMLRead_DidReadLastProperty_WithMissingProperty_IsFalse) + { + DoTextTransfer (); + } + + //------------------------------------------------------------------------- + +#define kDoubleValue 0.1 +#define kFloatValue -2.5f +#define kIntValue 1337 +#define kLongLongValue 1234567890123456789LL +#define kCharValue 'X' +#define kBoolValue true +#define kStringValue "UnityFTW" +#define kVectorSize 3 + +template +struct FloatingPointConsistencyTest +{ + #define kNumFloatValues 12 + T values[kNumFloatValues]; + + T Get(int i) + { + switch (i) + { + case 0: return std::numeric_limits::min(); + case 1: return std::numeric_limits::max(); + case 2: return std::numeric_limits::denorm_min(); + case 3: return std::numeric_limits::infinity(); + case 4: return -std::numeric_limits::infinity(); + case 5: return std::numeric_limits::quiet_NaN(); + case 6: return std::numeric_limits::epsilon(); + case 7: return -0.0; + case 8: return (T)12345678901234567890.123456789012345678900; + case 9: return (T)0.1; + case 10: return (T)(1.0 / 3.0); + case 11: return (T)(3 * 1024 * 1024 * 0.19358); + default: ErrorString("Should not happen!"); return 0; + } + } + + void FillStruct () + { + for (int i=0; i +template inline +void FloatingPointConsistencyTest::Transfer (TransferFunction& transfer) +{ + for (int i=0; i m_FloatTest; + FloatingPointConsistencyTest m_DoubleTest; + UnityStr m_String; + TestStruct m_Struct; + std::vector m_Vector; + std::map m_Map; + char m_TypelessData[kVectorSize]; + + void FillStruct () + { + m_FloatTest.FillStruct(); + m_DoubleTest.FillStruct(); + m_String = kStringValue; + m_Struct.FillStruct(); + m_Vector.resize(kVectorSize); + for (int i=0;i inline +void TestStruct::Transfer (TransferFunction& transfer) +{ + TRANSFER(m_Float); + TRANSFER(m_Int); + TRANSFER(m_Char); + TRANSFER(m_Bool); + TRANSFER(m_LongLong); +} + +template inline +void TestStruct2::Transfer (TransferFunction& transfer) +{ + TRANSFER(m_FloatTest); + TRANSFER(m_DoubleTest); + TRANSFER(m_String); + TRANSFER(m_Struct); + TRANSFER(m_Vector); + + TRANSFER(m_Map); + transfer.TransferTypelessData (kVectorSize, m_TypelessData, 0); +} + +#if SUPPORT_TEXT_SERIALIZATION +TEST (SerialializeYAMLStruct) +{ + TestStruct2 input; + input.FillStruct (); + + YAMLWrite write (0); + input.Transfer( write ); + std::string str; + write.OutputToString(str); + TestStruct2 output; + + YAMLRead read (str.c_str(), str.size(), 0); + output.Transfer( read ); + + output.VerifyStruct(); + +} //TEST +#endif + +} //SUITE + +#endif diff --git a/Runtime/Serialize/SerializeConversion.h b/Runtime/Serialize/SerializeConversion.h new file mode 100644 index 0000000..782ea5b --- /dev/null +++ b/Runtime/Serialize/SerializeConversion.h @@ -0,0 +1,34 @@ +#ifndef SERIALIZECONVERSION_H +#define SERIALIZECONVERSION_H + +#if SUPPORT_SERIALIZED_TYPETREES +#include "Runtime/Serialize/TransferFunctions/SerializeTransfer.h" +#include "SerializeTraits.h" + +// Trys to convert from an old type to a new one +template +bool StdTemplateConversionFunction (void* inData, SafeBinaryRead& transfer) +{ + NewFormat& data = *reinterpret_cast (inData); + const TypeTree& oldTypeTree = transfer.GetActiveOldTypeTree (); + AssertIf (SerializeTraits::GetTypeString (NULL) != oldTypeTree.m_Type); + OldFormat oldData; + + SafeBinaryRead safeRead; + CachedReader& temp = safeRead.Init (transfer); + + safeRead.Transfer (oldData, oldTypeTree.m_Name.c_str ()); + + temp.End (); + + data = oldData; + + return true; +} + +#define REGISTER_CONVERTER(from, to) \ +SafeBinaryRead::RegisterConverter (SerializeTraits::GetTypeString (NULL), SerializeTraits::GetTypeString (NULL), \ + StdTemplateConversionFunction) + +#endif +#endif diff --git a/Runtime/Serialize/SerializeTraits.h b/Runtime/Serialize/SerializeTraits.h new file mode 100644 index 0000000..9d5b946 --- /dev/null +++ b/Runtime/Serialize/SerializeTraits.h @@ -0,0 +1,533 @@ +#ifndef SERIALIZETRAITS_H +#define SERIALIZETRAITS_H + +#include "TypeTree.h" +#include "Runtime/Utilities/LogAssert.h" +#include "SerializeUtility.h" +#include "SerializationMetaFlags.h" +#include "Runtime/Utilities/vector_utility.h" +#include "Runtime/Utilities/vector_map.h" +#include "Runtime/Utilities/vector_set.h" +#include "Runtime/Utilities/dense_hash_map.h" +#include "Runtime/Utilities/dynamic_array.h" +#include +#include +#include +#include +#include "SerializeTraitsBase.h" + +class SerializedFile; +class SafeBinaryRead; + +typedef void TransferTypelessCallback (UInt8* data, int byteSize, int instanceID, int userdata); + +/* + + You can use SerializeTraits to setup transfer functions for classes where you can't change to code, eg. the STL. + You might also want to use it when writing custom converters. + + template<> + class SerializeTraits : public SerializeTraitsBase + { + public: + + typedef Vector4f value_type; + + inline static const char* GetTypeString () { return value_type::GetTypeString (); } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + data.Transfer (transfer); + } + + template + static void Convert (value_type& data, TransferFunction& transfer) + { + const TypeTree& oldTypeTree = transfer.GetActiveOldTypeTree (); + const std::string& oldType = transfer.GetActiveOldTypeTree ().m_Type; + if (oldType == "Vector3f") + { + Vector3f temp = data; + temp.Transfer (transfer); + data = temp; + return true; + } + else + return false; + } + + /// Returns whether or not a this type is to be treated as a seperate channel in the animation system + static bool IsAnimationChannel () { return T::IsAnimationChannel (); } + }; + +*/ + + +#define DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS(x) \ + inline static const char* GetTypeString (void* p = 0) { return #x; } \ + inline static bool IsAnimationChannel () { return true; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (float) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (double) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + // We use "int" rather than "SInt32" here for backwards-compatibility reasons. + // "SInt32" and "int" used to be two different types (as were "UInt32" and "unsigned int") + // that we now serialize through same path. We use "int" instead of "SInt32" as the common + // identifier as it was more common. + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (int) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (unsigned int) // See definition of "int" above. +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt64) +}; +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt64) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt16) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt16) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt8) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt8) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBaseForBasicType +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (char) +}; + +template<> +struct SerializeTraits : public SerializeTraitsBase +{ + typedef bool value_type; + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (bool) + + static int GetByteSize () { return 1; } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + #if (defined __ppc__) && !UNITY_WII + AssertIf (sizeof(bool) != 4); + UInt8& temp = *(reinterpret_cast(&data) + 3); + + transfer.TransferBasicData (temp); + + // When running in debug mode in OS X (-O0 in gcc), + // bool values which are not exactly 0x01 are treated as false. + // We don't want this. Cast UInt8 to bool to fix this. + if (transfer.IsReading()) + data = temp; + #if DEBUGMODE + AssertIf((transfer.IsReading() || transfer.IsWriting()) && (reinterpret_cast (data) != 0 && reinterpret_cast (data) != 1)); + #endif + #else + AssertIf (sizeof(bool) != 1); + UInt8& temp = reinterpret_cast(data); + transfer.TransferBasicData (temp); + + // When running in debug mode in OS X (-O0 in gcc), + // bool values which are not exactly 0x01 are treated as false. + // We don't want this. Cast UInt8 to bool to fix this. + #if DEBUGMODE + if (transfer.IsReading()) + data = temp; + // You constructor or Reset function is not setting the bool value to a defined value! + AssertIf((transfer.IsReading() || transfer.IsWriting()) && (temp != 0 && temp != 1)); + #endif + #endif + } +}; + + + +#define DEFINE_GET_TYPESTRING_MAP_CONTAINER(x) \ +inline static const char* GetTypeString (void*) { return #x; } \ +inline static bool IsAnimationChannel () { return false; } \ +inline static bool MightContainPPtr () { return SerializeTraits::MightContainPPtr() || SerializeTraits::MightContainPPtr(); } \ +inline static bool AllowTransferOptimization () { return false; } + +template<> +class SerializeTraits : public SerializeTraitsBase +{ +public: + + typedef UnityStr value_type; + inline static const char* GetTypeString (value_type* x = NULL) { return "string"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return false; } + inline static bool AllowTransferOptimization () { return false; } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data, kHideInEditorMask); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + + static void ResizeSTLStyleArray (value_type& data, int rs) + { + data.resize (rs, 1); + } + +}; + +// Do not add this serialization function. All serialized strings should use UnityStr instead of std::string +//template +//class SerializeTraits > : public SerializeTraitsBase > + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::vector value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { resize_trimmed (data, rs); } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::vector value_type; + + inline static const char* GetTypeString (void* x = NULL) { return "vector"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return false; } + inline static bool AllowTransferOptimization () { return false; } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { resize_trimmed (data, rs); } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::list value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return false; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize (rs); } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::pair value_type; + inline static const char* GetTypeString (void* x = NULL) { return "pair"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return SerializeTraits::MightContainPPtr() || SerializeTraits::MightContainPPtr(); } +// inline static bool AllowTransferOptimization () { return SerializeTraits::AllowTransferOptimization() || SerializeTraits::AllowTransferOptimization(); } + inline static bool AllowTransferOptimization () { return false; } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.Transfer (data.first, "first"); + transfer.Transfer (data.second, "second"); + } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::map value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef dense_hash_map value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::multimap value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef std::set value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef vector_map value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER (map) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + + + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef vector_map value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER (map) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef vector_set value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + + typedef vector_set value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + + +// Vector serialization is not allowed +template +class SerializeTraits > : public SerializeTraitsBase > +{ + public: + // disallow vector serialization +}; + + +template +class SerializeTraits > : public SerializeTraitsBase > +{ +public: + + typedef dynamic_array value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize_initialized(rs); } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + data.assign_external(reinterpret_cast (begin), reinterpret_cast (end)); + } +}; + +template<> +class SerializeTraits > : public SerializeTraitsBase > +{ +public: + + typedef dynamic_array value_type; + typedef UInt8 T; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize_initialized(rs); } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + data.assign_external(reinterpret_cast (begin), reinterpret_cast (end)); + } +}; + + +template +struct NonConstContainerValueType +{ + typedef typename T::value_type value_type; +}; + +template +struct NonConstContainerValueType > +{ + typedef T value_type; +}; + +template +struct NonConstContainerValueType > +{ + typedef std::pair value_type; +}; + +template +struct NonConstContainerValueType > +{ + typedef std::pair value_type; +}; + +template +struct NonConstContainerValueType > +{ + typedef std::pair value_type; +}; + +#endif diff --git a/Runtime/Serialize/SerializeTraitsBase.h b/Runtime/Serialize/SerializeTraitsBase.h new file mode 100644 index 0000000..b3a85fd --- /dev/null +++ b/Runtime/Serialize/SerializeTraitsBase.h @@ -0,0 +1,61 @@ +#pragma once + +template +class SerializeTraitsBase +{ + public: + + typedef T value_type; + + static int GetByteSize () {return sizeof (value_type);} + static size_t GetAlignOf() {return ALIGN_OF(value_type);} + + static void resource_image_assign_external (value_type& /*data*/, void* /*begin*/, void* /*end*/) + { + AssertString("Unsupported"); + } +}; + +template +class SerializeTraitsBaseForBasicType : public SerializeTraitsBase +{ +public: + typedef T value_type; + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferBasicData (data); + } +}; + +template +class SerializeTraits : public SerializeTraitsBase +{ +public: + + typedef T value_type; + + inline static const char* GetTypeString (void* /*ptr*/) { return value_type::GetTypeString (); } + inline static bool MightContainPPtr () { return value_type::MightContainPPtr (); } + /// Returns whether or not a this type is to be treated as a seperate channel in the animation system + static bool IsAnimationChannel () { return T::IsAnimationChannel (); } + + /// AllowTransferOptimization can be used for type that have the same memory format as serialized format. + /// Eg. a float or a Vector3f. + /// StreamedBinaryRead will collapse the read into a direct read when reading an array with values that have AllowTransferOptimization. + static bool AllowTransferOptimization () { return T::AllowTransferOptimization (); } + + template inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + data.Transfer (transfer); + } + +}; + +#define DEFINE_GET_TYPESTRING_CONTAINER(x) \ +inline static const char* GetTypeString (void*) { return #x; } \ +inline static bool IsAnimationChannel () { return false; } \ +inline static bool MightContainPPtr () { return SerializeTraits::MightContainPPtr(); } \ +inline static bool AllowTransferOptimization () { return false; } diff --git a/Runtime/Serialize/SerializeUtility.h b/Runtime/Serialize/SerializeUtility.h new file mode 100644 index 0000000..086f0e3 --- /dev/null +++ b/Runtime/Serialize/SerializeUtility.h @@ -0,0 +1,95 @@ +#ifndef SERIALIZEUTILITY_H +#define SERIALIZEUTILITY_H + + +#include "SerializationMetaFlags.h" + +#define TRANSFER(x) transfer.Transfer (x, #x) +#define TRANSFER_SIMPLE(x) transfer.Transfer (x, #x, kSimpleEditorMask) + +#if UNITY_EDITOR +#define TRANSFER_EDITOR_ONLY(x) if (!transfer.IsSerializingForGameRelease()) { transfer.Transfer (x, #x, kDontAnimate); } +#else +#define TRANSFER_EDITOR_ONLY(x) { } +#endif + +#if UNITY_EDITOR +#define TRANSFER_EDITOR_ONLY_HIDDEN(x) if (!transfer.IsSerializingForGameRelease()) { transfer.Transfer (x, #x, kHideInEditorMask); } +#else +#define TRANSFER_EDITOR_ONLY_HIDDEN(x) { } +#endif + +#define TRANSFER_WITH_CUSTOM_GET_SET(TYPE, STR_NAME, GET, SET, OPTIONS) \ + { \ + TYPE value; \ + if (transfer.IsWriting ()) { GET ; } \ + transfer.Transfer(value, STR_NAME, OPTIONS); \ + if (transfer.DidReadLastProperty ()) { SET ; } \ + } + +#define TRANSFER_PROPERTY(TYPE,NAME,GET,SET)\ + TRANSFER_WITH_CUSTOM_GET_SET(TYPE, #NAME, value = GET (), SET (value), kNoTransferFlags) + +#define TRANSFER_ENUM(x) { Assert(sizeof(x) == sizeof(int)); transfer.Transfer ((int&)x, #x); } + +#if UNITY_EDITOR +#define TRANSFER_DEBUG(x) { if (transfer.GetFlags () & kSerializeDebugProperties) transfer.Transfer (x, #x, kDebugPropertyMask | kNotEditableMask); } +#else +#define TRANSFER_DEBUG(x) +#endif + +template +inline bool SerializePrefabIgnoreProperties (T& transfer) +{ + return (transfer.GetFlags() & kSerializeForPrefabSystem) == 0; +} + +/// Usage: TRANSFER_PROPERTY_DEBUG(bool, m_Enabled, data->GetEnabled) +#define TRANSFER_PROPERTY_DEBUG(TYPE,NAME,GET) \ +if (transfer.GetFlags () & kSerializeDebugProperties){\ + TYPE NAME;\ + if (transfer.IsWriting ())\ + NAME = GET ();\ + transfer.Transfer (NAME, #NAME, kDebugPropertyMask | kNotEditableMask);\ +} + + +#define DEFINE_GET_TYPESTRING(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return true; } \ + inline static bool AllowTransferOptimization () { return false; } + +#define DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return true; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } + + +#define DECLARE_SERIALIZE(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return true; } \ + inline static bool AllowTransferOptimization () { return false; } \ + template \ + void Transfer (TransferFunction& transfer); + +#define DECLARE_SERIALIZE_NO_PPTR(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return false; } \ + template \ + void Transfer (TransferFunction& transfer); + +#define DECLARE_SERIALIZE_OPTIMIZE_TRANSFER(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } \ + template \ + void Transfer (TransferFunction& transfer); + + +#endif diff --git a/Runtime/Serialize/SerializedFile.cpp b/Runtime/Serialize/SerializedFile.cpp new file mode 100644 index 0000000..26a0a3b --- /dev/null +++ b/Runtime/Serialize/SerializedFile.cpp @@ -0,0 +1,1520 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" +#include "SerializedFile.h" +#include "Runtime/Utilities/Utility.h" +#include "SerializeConversion.h" +#include "TransferUtility.h" +#include "Runtime/Serialize/TransferFunctions/TransferNameConversions.h" +#include "CacheWrap.h" +#include "Runtime/Utilities/Word.h" +#include "Configuration/UnityConfigureVersion.h" +#include "BuildTargetVerification.h" +#include "Runtime/Utilities/FileUtilities.h" +#if UNITY_WII +#include "PlatformDependent/Wii/WiiUtility.h" +#include "PlatformDependent/Wii/WiiLoadingScreen.h" +#endif + +#include "Runtime/Misc/Allocator.h" + +/// Set this to 1 to dump type trees to the console when they don't match between +/// the runtime and the file that is being loaded. This is most useful when debugging +/// loading issues in players (also enable DEBUG_FORCE_ALWAYS_WRITE_TYPETREES in +/// BuildPlayerUtility.cpp to have type trees included in player data). +#define DEBUG_LOG_TYPETREE_MISMATCHES 0 + +using namespace std; + +enum { kCurrentSerializeVersion = 9 }; + +const char* kAssetBundleVersionNumber = "1"; +const char* kUnityTextMagicString = "%YAML 1.1"; +#define kUnityTextHeaderFileID -1 + +bool IsSerializedFileTextFile(string pathName) +{ + const int magiclen = strlen(kUnityTextMagicString); + + char compare[256]; + if (!ReadFromFile (pathName, compare, 0, magiclen)) + return false; + + compare[magiclen] = '\0'; + if (strcmp(compare, kUnityTextMagicString) == 0) + return true; + + return false; +} + + +#if ENABLE_SECURITY +#define TEST_LEN(x) if (iterator + sizeof(x) > end) \ +{\ + return false; \ +} + +#define TEST_READ_SIZE(x) if (iterator + x > end) \ +{\ +return false; \ +} + +#else +#define TEST_LEN(x) +#define TEST_READ_SIZE(x) +#endif + +static const int kHeaderSize_Ver8 = 12; +static const int kPreallocateFront = 4096; + +struct SerializedFileHeader +{ + // This header is always in BigEndian when in file + // Metadata follows directly after the header + UInt32 m_MetadataSize; + UInt32 m_FileSize; + UInt32 m_Version; + UInt32 m_DataOffset; + UInt8 m_Endianess; + UInt8 m_Reserved[3]; + + void SwapEndianess () + { + SwapEndianBytes (m_MetadataSize); + SwapEndianBytes (m_FileSize); + SwapEndianBytes (m_Version); + SwapEndianBytes (m_DataOffset); + } +}; + +int RemapClassIDToNewClassID (int classID) +{ + switch(classID) + { + case 1012: return 1011; // AvatarSkeletonMask -> AvatarMask + default: return classID; + } +} + +SerializedFile::SerializedFile () +: m_Externals(1024,kMemSerialization) +{ + m_ReadOffset = 0; + m_WriteDataOffset = 0; + m_IsDirty = false; + m_MemoryStream = false; + m_HasErrors = false; + m_CachedFileStream = false; + m_TargetPlatform = kBuildNoTargetPlatform; + m_SubTarget = 0; + + #if SUPPORT_TEXT_SERIALIZATION + m_IsTextFile = false; + #endif + + #if SUPPORT_SERIALIZE_WRITE + m_CachedWriter = NULL; + #endif + + m_ReadFile = NULL; +} + + +#if SUPPORT_SERIALIZE_WRITE +bool SerializedFile::InitializeWrite (CachedWriter& cachedWriter, BuildTargetSelection target, int options) +{ + SET_ALLOC_OWNER(this); + m_TargetPlatform = target.platform; + m_SubTarget = target.subTarget; + + m_CachedWriter = &cachedWriter; + + Assert (!((options & kAllowTextSerialization) && (options & kSerializeGameRelease))); + m_IsTextFile = options & kAllowTextSerialization; + + if (!m_IsTextFile) + { + void* buffer = alloca (kPreallocateFront); + memset (buffer, 0, kPreallocateFront); + + // Write header and reserve space for metadata. In case the resulting metadata will not fit + // in the preallocated space we'll remove it and write it tightly packed in FinalizeWrite later. + // In case it fits, we'll have a hole between meta and object data and that's fine. + + m_CachedWriter->Write(buffer, kPreallocateFront); + m_WriteDataOffset = m_CachedWriter->GetPosition (); + } + + return FinalizeInit(options); +} +#endif + +bool SerializedFile::InitializeRead (const string& path, ResourceImageGroup& resourceImage, unsigned cacheSize, unsigned cacheCount, int options, int readOffset) +{ + SET_ALLOC_OWNER(this); + m_ReadOffset = readOffset; + m_ReadFile = UNITY_NEW( FileCacherRead (path, cacheSize, cacheCount), kMemFile); + m_ResourceImageGroup = resourceImage; + + return FinalizeInit(options); +} + +bool SerializedFile::InitializeMemoryBlocks (const string& path, UInt8** buffer, unsigned size, unsigned offset, int options) +{ + SET_ALLOC_OWNER(this); + m_MemoryStream = true; + m_ReadOffset = offset; + m_ReadFile = UNITY_NEW( MemoryCacherReadBlocks (buffer, size, kCacheBlockSize), kMemFile); + + return FinalizeInit(options); +} + +bool SerializedFile::FinalizeInit (int options) +{ + m_Options = options; + #if GAMERELEASE + m_Options |= kSerializeGameRelease; + #endif + + if (m_Options & kSwapEndianess) + m_FileEndianess = kOppositeEndianess; + else + m_FileEndianess = kActiveEndianess; + + if (m_ReadFile) + { +#if SUPPORT_TEXT_SERIALIZATION + const int magiclen = strlen(kUnityTextMagicString); + char compare[256]; + if (m_ReadFile->GetFileLength () >= magiclen) + { + ReadFileCache (*m_ReadFile, compare, 0 + m_ReadOffset, magiclen); + compare[magiclen] = '\0'; + if (strcmp(compare, kUnityTextMagicString) == 0) + { + m_IsTextFile = true; + m_ReadOffset += magiclen; + return ReadHeaderText(); + } + } + m_IsTextFile = false; +#endif + return ReadHeader(); + } + else + { +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + string label = kUnityTextMagicString; + label += "\n%TAG !u! tag:unity3d.com,2011:\n"; + m_CachedWriter->Write (&label[0], label.length()); + } +#endif + return true; +} +} + +SerializedFile::~SerializedFile () +{ + UNITY_DELETE( m_ReadFile, kMemFile); +} + + +#if SUPPORT_SERIALIZE_WRITE +bool SerializedFile::FinishWriting () +{ + AssertIf(m_CachedWriter == NULL); + + if (m_CachedWriter != NULL) + { + if (!m_IsTextFile) + { + SerializationCache metadataBuffer; + + if (!ShouldSwapEndian()) + { + BuildMetadataSection (metadataBuffer, m_WriteDataOffset); + return WriteHeader (metadataBuffer); + } + else + { + BuildMetadataSection (metadataBuffer, m_WriteDataOffset); + return WriteHeader (metadataBuffer); + } + } + else + { + bool success = m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase().WriteHeaderAndCloseFile(NULL, 0, 0); + return success; + } + } + + return false; +} + +static void WriteAlignmentData (File& file, size_t misalignment) +{ + Assert (misalignment < SerializedFile::kSectionAlignment); + UInt8 data[SerializedFile::kSectionAlignment]; + memset (data, 0, misalignment); + file.Write(data, misalignment); +} + +template +bool SerializedFile::WriteHeader (SerializationCache& metadata) +{ + bool success = true; + + // The aggregated metadata fits into the pre-written block, so write it directly. + if (metadata.size () <= kPreallocateFront - sizeof (SerializedFileHeader)) + { + UInt8* temp = (UInt8*)alloca (kPreallocateFront); + + SerializedFileHeader& header = *(SerializedFileHeader*)temp; + header.m_MetadataSize = metadata.size (); + header.m_FileSize = m_CachedWriter->GetPosition (); + header.m_Version = kCurrentSerializeVersion; + header.m_DataOffset = m_WriteDataOffset; + header.m_Endianess = m_FileEndianess; + memset (header.m_Reserved, 0, sizeof header.m_Reserved); + + if (kActiveEndianess != kBigEndian) + header.SwapEndianess (); + + std::copy (metadata.begin (), metadata.end (), temp + sizeof (SerializedFileHeader)); + success &= m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase ().WriteHeaderAndCloseFile (temp, 0, sizeof (SerializedFileHeader) + metadata.size ()); + } + else + { + // metadata doesn't fit, therefore close the file, write header + metadata to another file + // and copy data over from 'this' one. + + success &= m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase ().WriteHeaderAndCloseFile (NULL, 0, 0); + + size_t dataFileSize = m_CachedWriter->GetPosition (); + if (dataFileSize < kPreallocateFront) + return false; + + size_t dataSize = dataFileSize - kPreallocateFront; + size_t dataOffsetOriginal = sizeof (SerializedFileHeader) + metadata.size (); + size_t dataOffset = RoundUp (dataOffsetOriginal, kSectionAlignment); + + std::string originalPath = m_CachedWriter->GetCacheBase().GetPathName (); + std::string tempPath = GenerateUniquePathSafe (originalPath); + + SerializedFileHeader header = + { + metadata.size (), dataOffset + dataSize, + kCurrentSerializeVersion, + dataOffset, + m_FileEndianess, 0, 0, 0 + }; + + if (kActiveEndianess != kBigEndian) + header.SwapEndianess (); + + File file; + success &= file.Open(tempPath, File::kWritePermission); + + // header + success &= file.Write (&header, sizeof (header)); + + // metadata + success &= file.Write (&*metadata.begin (), metadata.size ()); + if (dataOffset != dataOffsetOriginal) + WriteAlignmentData (file, dataOffset - dataOffsetOriginal); + FatalErrorIf (dataOffset != file.GetPosition ()); + + { + enum { kCopyChunck = 1 * 1024 * 1024 }; + + UInt8* buffer; + ALLOC_TEMP(buffer, UInt8, kCopyChunck); + + File srcFile; + success &= srcFile.Open(originalPath, File::kReadPermission); + + size_t position = kPreallocateFront; + size_t left = dataSize; + while (left > 0 && success) + { + size_t toRead = (std::min)((size_t)kCopyChunck, left); + int wasRead = srcFile.Read (position, buffer, toRead); + success &= file.Write (buffer, wasRead); + position += toRead; + left -= toRead; + } + success &= srcFile.Close (); + + success &= file.Close (); + } + + // move the temp file over to the destination + success &= DeleteFile(originalPath); + success &= MoveFileOrDirectory (tempPath, originalPath); + } + + return success; +} +#endif // SUPPORT_SERIALIZE_WRITE + +enum { kMaxTypeCount = 100000 }; + +bool SerializedFile::ReadHeader () +{ + AssertIf (m_ReadFile == NULL); + + SerializedFileHeader header; + + if (m_ReadFile->GetFileLength () < sizeof (header)) + return false; + + ReadFileCache (*m_ReadFile, &header, m_ReadOffset, sizeof (header)); + + if (kActiveEndianess == kLittleEndian) + header.SwapEndianess (); + + // Consistency check if the file is a valid serialized file. + if (header.m_MetadataSize == -1) + return false; + if (header.m_Version == 1) + return false; + if (header.m_Version > kCurrentSerializeVersion) + return false; + + unsigned metadataSize, metadataOffset; + unsigned dataSize, dataOffset; + unsigned dataEnd; + + if (header.m_Version >= 9) + { + // If we're reading a stream file, m_ReadOffset + header.m_FileSize will not necessarilly be equal to m_ReadFile->GetFileLength(), + // because there can be few padding bytes which doesn't count into header.m_FileSize + // See WriteStreamFile in BuildPlayerUtility.cpp + if ((m_ReadOffset + header.m_FileSize) > m_ReadFile->GetFileLength () || header.m_DataOffset > header.m_FileSize) + return false; + + // [header][metadata[...]][data] + + metadataOffset = sizeof header; + metadataSize = header.m_MetadataSize; + + m_FileEndianess = header.m_Endianess; + + dataOffset = header.m_DataOffset; + dataSize = header.m_FileSize - header.m_DataOffset; + dataEnd = dataOffset + dataSize; + } + else + { + // [header][data][metadata] + + // We set dataOffset to zero, because offsets in object table are file-start based + dataOffset = 0; + dataSize = header.m_FileSize - header.m_MetadataSize - kHeaderSize_Ver8; + dataEnd = header.m_FileSize - header.m_MetadataSize; + + // Offset by one, because we're reading the endianess flag right here + metadataOffset = header.m_FileSize - header.m_MetadataSize + 1; + metadataSize = header.m_MetadataSize - 1; + + if (metadataSize == -1 || (m_ReadOffset + header.m_FileSize) > m_ReadFile->GetFileLength () || dataEnd > header.m_FileSize) + return false; + + ReadFileCache (*m_ReadFile, &m_FileEndianess, m_ReadOffset + metadataOffset - 1, sizeof (m_FileEndianess)); + } + + // Check endianess validity + if (m_FileEndianess != kBigEndian && m_FileEndianess != kLittleEndian) + return false; + + SerializationCache metadataBuffer; + metadataBuffer.resize (metadataSize); + ReadFileCache (*m_ReadFile, &metadataBuffer[0], m_ReadOffset + metadataOffset, metadataSize); + + bool result; + if (m_FileEndianess == kActiveEndianess) + { + result = ReadMetadata(header.m_Version, dataOffset, &*metadataBuffer.begin (), metadataBuffer.size (), dataEnd); + } + else + { + result = ReadMetadata(header.m_Version, dataOffset, &*metadataBuffer.begin (), metadataBuffer.size (), dataEnd); + } + + if (!result) + { + ErrorString(Format("Failed to read file '%s' because it is corrupted.", m_ReadFile->GetPathName ().c_str())); + } + return result; +} + +#if SUPPORT_TEXT_SERIALIZATION +bool SerializedFile::IndexTextFile() +{ + const size_t kBufferLength = 1024; + const size_t kMaxLineLength = 256; + bool hasMergeConflicts = false; + string read; + read.resize (kBufferLength); + + size_t readPos = 0; + size_t lineStart = 0; + int lineCount = 1; //We start counting lines at one, not zero. + ObjectInfo *curInfo = NULL; + + const char *guidLabel = "guid: "; + int guidLabelLen = strlen (guidLabel); + int guidLabelPos = 0; + bool lineContainsGUID = false; + std::string prevLine; + std::set externals; + while (readPos < m_ReadFile->GetFileLength() - m_ReadOffset) + { + size_t readBufferLength = std::min (m_ReadFile->GetFileLength() - m_ReadOffset - readPos, kBufferLength); + ReadFileCache (*m_ReadFile, &read[0], m_ReadOffset + readPos, readBufferLength); + + for (size_t i=0; ibyteSize = lineStart - curInfo->byteStart; + SInt32 fileID, classID; + if (sscanf(line.c_str(), "--- !u!%d &%d", (int*)&classID, (int*)&fileID) == 2) + { + curInfo = &m_Object[fileID]; + curInfo->classID = RemapClassIDToNewClassID(classID); + curInfo->typeID = 0; + curInfo->byteStart = lineEnd; + curInfo->isDestroyed = false; + curInfo->debugLineStart = lineCount; + } + } + break; + #if UNITY_EDITOR + case '>': + case '<': + case '=': + if (!hasMergeConflicts) + WarningStringMsg ("The file %s seems to have merge conflicts. Please open it in a text editor and fix the merge.\n", m_DebugPath.c_str()); + hasMergeConflicts = true; + break; + #endif + } + if (lineContainsGUID) + { + if (line.find ('}') == string::npos) + prevLine = line; + else + { + line = prevLine + line; + line = line.substr(line.find ('{')); + YAMLRead read (line.c_str(), line.size(), 0, &m_DebugPath, lineCount-1); + + FileIdentifier id; + + read.Transfer (id.guid, "guid"); + read.Transfer (id.type, "type"); + id.Fix_3_5_BackwardsCompatibility(); + + if (id.guid != UnityGUID()) + externals.insert (id); + else + ErrorStringMsg ("Could not extract GUID in text file %s at line %d.", m_DebugPath.c_str(), lineCount-1); + + lineContainsGUID = false; + prevLine = ""; + } + } + } + lineStart = lineEnd; + } + } + + readPos += readBufferLength; + } + if (curInfo) + curInfo->byteSize = readPos - curInfo->byteStart; + + m_Externals.assign (externals.begin(), externals.end()); + return !hasMergeConflicts; +} + +bool SerializedFile::ReadHeaderText () +{ + Assert (m_ReadFile != NULL); + return IndexTextFile (); +} + +template +void SerializedFile::WriteTextSerialized (std::string &label, T &data, int options) +{ + Assert (m_CachedWriter != NULL); + + m_CachedWriter->Write (&label[0], label.length()); + + YAMLWrite writeStream (options, &m_DebugPath); + data.VirtualRedirectTransfer (writeStream); + writeStream.OutputToCachedWriter(m_CachedWriter); + if (writeStream.HasError()) + m_HasErrors = true; +} + +#endif + +// The header is put at the end of and is only allowed to be read at startup + +template +bool SerializedFile::ReadMetadata (int version, unsigned dataOffset, UInt8 const* data, size_t length, size_t dataFileEnd) +{ + AssertIf(kSwap && kActiveEndianess == m_FileEndianess); + AssertIf(!kSwap && kOppositeEndianess == m_FileEndianess); + SET_ALLOC_OWNER(this); + + UInt8 const* iterator = data, *end = data + length; + + // Read Unity version file was built with + UnityStr unityVersion; + if (version >= 7) + { + if (!ReadString(unityVersion, iterator, end)) + return false; + } + + // Build target platform verification + if (version >= 8) + { + TEST_LEN(m_TargetPlatform); + ReadHeaderCache (m_TargetPlatform, iterator); + + if (!CanLoadFileBuiltForTargetPlatform(static_cast(m_TargetPlatform))) + { + ErrorStringMsg( + "The file can not be loaded because it was created for another build target that is not compatible with this platform.\n" + "Please make sure to build asset bundles using the build target platform that it is used by.\n" + "File's Build target is: %d\n", + (int)m_TargetPlatform + ); + return false; + } + } + + // Read number of types + SInt32 typeCount; + TEST_LEN(typeCount); + ReadHeaderCache (typeCount, iterator); + + #if SUPPORT_SERIALIZED_TYPETREES + // Read types + for (int i=0;i (classID, iterator); + if (!ReadTypeTree (*readType, iterator, end, version, kSwap)) + return false; + classID = RemapClassIDToNewClassID(classID); + + m_Type[classID].SetOldType (readType); + } + #else + if (typeCount != 0) + { + ErrorString("Serialized file contains typetrees but the target can't use them. Will ignore typetrees."); + } + #endif + + SInt32 bigIDEnabled = 0; + if (version >= 7) + ReadHeaderCache (bigIDEnabled, iterator); + + // Read number of objects + SInt32 objectCount; + TEST_LEN(objectCount); + ReadHeaderCache (objectCount, iterator); + + // Check if the size is roughly out of bounds, we only want to prevent running out of memory due to insane objectCount value here. + TEST_READ_SIZE(objectCount * 12) + + // Read Objects + m_Object.reserve(objectCount); + for (int i=0;i LOCAL_IDENTIFIER_IN_FILE_SIZE); + UInt64 fileID64; + TEST_LEN(fileID64); + ReadHeaderCache (fileID64, iterator); + fileID = fileID64; + } + else + { + UInt32 fileID32; + TEST_LEN(fileID32); + ReadHeaderCache (fileID32, iterator); + fileID = fileID32; + } + + TEST_LEN(value); + ReadHeaderCache (value.byteStart, iterator); + ReadHeaderCache (value.byteSize, iterator); + ReadHeaderCache (value.typeID, iterator); + ReadHeaderCache (value.classID, iterator); + ReadHeaderCache (value.isDestroyed, iterator); + + value.byteStart += dataOffset; + + // TODO check this with joachim + value.typeID = RemapClassIDToNewClassID(value.typeID); + value.classID = RemapClassIDToNewClassID(value.classID); + + AssertIf (value.byteStart + value.byteSize > dataFileEnd); + if (value.byteStart < 0 || value.byteSize < 0 || value.byteStart + value.byteSize < value.byteStart || value.byteStart + value.byteSize > dataFileEnd) + return false; + + m_Object.push_unsorted(fileID, value); + //printf_console ("fileID: %d byteStart: %d classID: %d \n", fileID, value.byteStart, value.classID); + } + + // If there's no type tree then Unity version must mach exactly. + // + // For asset bundles we write the asset bundle serialize version and compare against that. + // The asset bundle itself contains hashes of all serialized classes and uses it to figure out if an asset bundle can be loaded. + bool needsVersionCheck = !m_Object.empty() && typeCount == 0 && (m_Options & kIsBuiltinResourcesFile) == 0; + if (needsVersionCheck) + { + bool versionPasses; + string::size_type newLinePosition = unityVersion.find('\n'); + // Compare Unity version number + if (newLinePosition == string::npos) + versionPasses = unityVersion == UNITY_VERSION; + // Compare asset bundle serialize version + else + versionPasses = string (unityVersion.begin() + newLinePosition + 1, unityVersion.end()) == kAssetBundleVersionNumber; + + if (!versionPasses) + { + ErrorStringMsg("Invalid serialized file version. File: \"%s\". Expected version: " UNITY_VERSION ". Actual version: %s.", m_ReadFile->GetPathName().c_str(), unityVersion.c_str()); + return false; + } + } + + #if SUPPORT_SERIALIZED_TYPETREES + if (unityVersion.find("3.5.0f5") == 0) + m_Options |= kWorkaround35MeshSerializationFuckup; + #endif + +// printf_console("file version: %s - '%s'\n", unityVersion.c_str(), m_DebugPath.c_str()); + + // Read externals/pathnames + SInt32 externalsCount; + TEST_LEN(externalsCount); + + ReadHeaderCache (externalsCount, iterator); + // Check if the size is roughly out of bounds, we only want to prevent running out of memory due to insane externalsCount value here. + TEST_READ_SIZE(externalsCount) + + m_Externals.resize (externalsCount); + + for (int i=0;i= 5) + { + if (version >= 6) + { + ///@TODO: Remove from serialized file format + UnityStr tempEmpty; + if (!ReadString(tempEmpty, iterator, end)) + return false; + } + + TEST_LEN(m_Externals[i].guid.data); + for (int g=0;g<4;g++) + ReadHeaderCache (m_Externals[i].guid.data[g], iterator); + + TEST_LEN(m_Externals[i].type); + ReadHeaderCache (m_Externals[i].type, iterator); + if (!ReadString (m_Externals[i].pathName, iterator, end)) + return false; + } + else + { + if (!ReadString (m_Externals[i].pathName, iterator, end)) + return false; + } + + #if UNITY_EDITOR + m_Externals[i].Fix_3_5_BackwardsCompatibility (); + #endif + + m_Externals[i].CheckValidity (); + } + + // Read Userinfo string + if (version >= 5) + { + UnityStr userInformation; + if (!ReadString (userInformation, iterator, end)) + return false; + } + + Assert (iterator == end); + + return true; +} + +#if SUPPORT_SERIALIZE_WRITE + +template +void SerializedFile::BuildMetadataSection (SerializationCache& cache, unsigned dataOffsetInFile) +{ + // Write Unity version file is being built with + UnityStr version = UNITY_VERSION; + if (m_Options & kSerializedAssetBundleVersion) + { + version += "\n"; + version += kAssetBundleVersionNumber; + } + WriteString (version, cache); + + WriteHeaderCache (m_TargetPlatform, cache); + + if ((m_Options & kDisableWriteTypeTree) == 0) + { + // Write number of types + SInt32 typeCount = m_Type.size (); + WriteHeaderCache (typeCount, cache); + + // Write type data + for (TypeMap::iterator i = m_Type.begin ();i != m_Type.end ();i++) + { + AssertIf (i->second.GetOldType () == NULL); + WriteHeaderCache (i->first, cache); + WriteTypeTree (*i->second.GetOldType (), cache, kSwap); + } + } + else + { + SInt32 typeCount = 0; + WriteHeaderCache (typeCount, cache); + } + + SInt32 bigIDEnabled = LOCAL_IDENTIFIER_IN_FILE_SIZE > 32; + WriteHeaderCache (bigIDEnabled, cache); + + // Write number of objects + SInt32 objectCount = m_Object.size (); + WriteHeaderCache (objectCount, cache); + for (ObjectMap::iterator i = m_Object.begin ();i != m_Object.end ();i++) + { + if (bigIDEnabled) + { + UInt64 bigID = i->first; + WriteHeaderCache (bigID, cache); + } + else + { + UInt32 smallID = i->first; + WriteHeaderCache (smallID, cache); + } + + WriteHeaderCache (i->second.byteStart - dataOffsetInFile, cache); + WriteHeaderCache (i->second.byteSize, cache); + WriteHeaderCache (i->second.typeID, cache); + WriteHeaderCache (i->second.classID, cache); + WriteHeaderCache (i->second.isDestroyed, cache); + + //printf_console ("fileID: %d byteStart: %d classID: %d \n", i->first, i->second.byteStart, i->second.classID); + } + + // Write externals + objectCount = m_Externals.size (); + WriteHeaderCache (objectCount, cache); + for (int i=0;i (m_Externals[i].guid.data[g], cache); + WriteHeaderCache (m_Externals[i].type, cache); + WriteString (m_Externals[i].pathName, cache); + } + + // Write User info + UnityStr tempUserInformation; + WriteString (tempUserInformation, cache); +} +#endif + +bool SerializedFile::IsAvailable (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + if (i == m_Object.end ()) + return false; + else + return ! i->second.isDestroyed; +} + +int SerializedFile::GetClassID (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.classID; +} + +int SerializedFile::GetByteStart (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.byteStart; +} + +int SerializedFile::GetByteSize (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.byteSize; +} +#if SUPPORT_SERIALIZED_TYPETREES +const TypeTree* SerializedFile::GetTypeTree (LocalIdentifierInFileType id) +{ + ObjectMap::iterator found = m_Object.find(id); + if (found == m_Object.end()) + return NULL; + + TypeMap::iterator type = m_Type.find (found->second.typeID); + if (type == m_Type.end ()) + return NULL; + return type->second.GetOldType (); +} +#endif + +#if !UNITY_EXTERNAL_TOOL +// objects: On return, all fileIDs to all objects in this Serialze +void SerializedFile::GetAllFileIDs (vector* objects)const +{ + AssertIf (objects == NULL); + + objects->reserve (m_Object.size ()); + ObjectMap::const_iterator i; + for (i=m_Object.begin ();i!=m_Object.end ();i++) + { + if (i->second.isDestroyed) + continue; + + Object::RTTI* rtti = Object::ClassIDToRTTI(i->second.classID); + if (rtti == NULL || rtti->factory == NULL) + continue; + + objects->push_back (i->first); + } +} +#endif +// objects: On return, all fileIDs to all objects in this Serialze +void SerializedFile::GetAllFileIDsUnchecked (vector* objects)const +{ + AssertIf (objects == NULL); + + objects->reserve (m_Object.size ()); + ObjectMap::const_iterator i; + for (i=m_Object.begin ();i!=m_Object.end ();i++) + { + if (i->second.isDestroyed) + continue; + + objects->push_back (i->first); + } +} + +LocalIdentifierInFileType SerializedFile::GetHighestID () const +{ + if (m_Object.empty ()) + return 0; + else + return m_Object.rbegin ()->first; +} + +// Returns whether or not the object referenced by id was destroyed +bool SerializedFile::DestroyObject (LocalIdentifierInFileType id) +{ + #if SUPPORT_SERIALIZE_WRITE + AssertIf(m_CachedWriter); + #endif + m_IsDirty = true; + + ObjectMap::iterator o; + o = m_Object.find (id); + if (o == m_Object.end ()) + { + SET_ALLOC_OWNER(this); + m_Object[id].isDestroyed = true; + return false; + } + else + { + o->second.isDestroyed = true; + return true; + } +} + +#if SUPPORT_SERIALIZE_WRITE + +void SerializedFile::WriteObject (Object& object, LocalIdentifierInFileType fileID, const BuildUsageTag& buildUsage) +{ + AssertIf (m_CachedWriter == NULL); + SET_ALLOC_OWNER(this); + + bool perObjectTypeTree = object.GetNeedsPerObjectTypeTree (); + + int typeID = object.GetClassID (); + + int mask = kNeedsInstanceIDRemapping | m_Options; + +#if UNITY_EDITOR + object.SetFileIDHint (fileID); +#endif + +#if SUPPORT_TEXT_SERIALIZATION + if (!m_IsTextFile) + { +#endif + + // Native C++ object typetrees share typetree by class id + if (!perObjectTypeTree) + { + // Include Type + Type& type = m_Type[typeID]; + + // If we need a perObjectTypeTree there is only one object using it (the one we are writing now) + // Thus we can safely replace the old typeTree + // If we have a per class typetree do not override the old typetree + if (type.GetOldType () == NULL) + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + GenerateTypeTree (object, typeTree, mask | kDontRequireAllMetaFlags); + + // Register the type tree + type.SetOldType (typeTree); + } + } + // Scripted objects we search the registered typetrees for duplicates and share + // or otherwise allocate a new typetree. + else + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + + GenerateTypeTree (object, typeTree, mask | kDontRequireAllMetaFlags); + + typeID = 0; + + // Find if there + for (TypeMap::iterator i=m_Type.begin();i!=m_Type.end();i++) + { + if (i->first > 0) + break; + + if (IsStreamedBinaryCompatbile(*typeTree, *i->second.GetOldType())) + { + typeID = i->first; + UNITY_DELETE(typeTree, kMemTypeTree); + break; + } + } + + // Allocate type id + if (typeID == 0) + { + if (m_Type.empty()) + typeID = -1; + else if (m_Type.begin()->first < 0) + typeID = m_Type.begin()->first - 1; + else + typeID = -1; + + Type& type = m_Type[typeID]; + + // Create new type and init it using a proxy transfer + // Register the type tree + type.SetOldType (typeTree); + } + } + +#if SUPPORT_TEXT_SERIALIZATION + } +#endif + + // We are not taking care of fragmentation. + const unsigned kFileAlignment = 8; + + unsigned unalignedByteStart = m_CachedWriter->GetPosition(); + + // Align the object to a kFileAlignment byte boundary + unsigned alignedByteStart = unalignedByteStart; + if (unalignedByteStart % kFileAlignment != 0) + alignedByteStart += kFileAlignment - unalignedByteStart % kFileAlignment; + + + AssertIf (m_Object.find (fileID) != m_Object.end () && m_Object.find (fileID)->second.classID != object.GetClassID ()); + ObjectInfo& info = m_Object[fileID]; + info.byteStart = alignedByteStart; + info.classID = object.GetClassID (); + info.isDestroyed = false; + info.typeID = typeID; + +/* ////// PRINT OUT serialized Data as ascii to console + if (false && gPrintfDataHack) + { + printf_console ("\n\nPrinting object: %d\n", fileID); + + // Set write marker to end of file and register the objects position in file + StreamedTextWrite writeStream; + CachedWriter& cache = writeStream.Init (kNeedsInstanceIDRemapping); + cache.Init (m_FileCacher, alignedByteStart, 0, false); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + cache.End (); + } +*/ + +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + string label = Format("--- !u!%d &%d\n", info.classID, (int)fileID); + WriteTextSerialized (label, object, mask); + } + else +#endif + if (!ShouldSwapEndian()) + { + // Set write marker to end of file and register the objects position in file + StreamedBinaryWrite writeStream; + + CachedWriter& cache = writeStream.Init (*m_CachedWriter, mask, BuildTargetSelection(static_cast(m_TargetPlatform),m_SubTarget), buildUsage); + char kZeroAlignment[kFileAlignment] = {0, 0, 0, 0, 0, 0, 0, 0}; + cache.Write (kZeroAlignment, alignedByteStart - unalignedByteStart); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + + *m_CachedWriter = cache; + } + else + { + // Set write marker to end of file and register the objects position in file + StreamedBinaryWrite writeStream; + CachedWriter& cache = writeStream.Init (*m_CachedWriter, mask, BuildTargetSelection(static_cast(m_TargetPlatform),m_SubTarget), buildUsage); + char kZeroAlignment[kFileAlignment] = {0, 0, 0, 0, 0, 0, 0, 0}; + cache.Write (kZeroAlignment, alignedByteStart - unalignedByteStart); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + + *m_CachedWriter = cache; + } + + info.byteSize = m_CachedWriter->GetPosition() - info.byteStart; +} +#endif + +#if !UNITY_EXTERNAL_TOOL + +enum { kMonoBehaviourClassID = 114 }; +static void OutOfBoundsReadingError (int classID, int expected, int was) +{ + if (classID == kMonoBehaviourClassID) + { + // This code should not access any member variables + // because that might dereference pointers etc during threaded loading. + ErrorString(Format("A script behaviour has a different serialization layout when loading. (Read %d bytes but expected %d bytes)\nDid you #ifdef UNITY_EDITOR a section of your serialized properties in any of your scripts?", was, expected)); + } + else + { + ErrorString(Format("Mismatched serialization in the builtin class '%s'. (Read %d bytes but expected %d bytes)", Object::ClassIDToString(classID).c_str(), was, expected)); + } +} + +void SerializedFile::ReadObject (LocalIdentifierInFileType fileID, int instanceId, ObjectCreationMode mode, bool isPersistent, TypeTree** oldTypeTree, bool* didChangeTypeTree, Object** outObjectPtr) +{ +#if UNITY_WII + wii::ProcessShutdownAndReset(); + wii::ProcessLoadingScreen(); +#endif +// printf_console("Reading instance: %d fileID: %d filePtr: %d\n", instanceId, fileID, this); + + ObjectMap::iterator iter = m_Object.find (fileID); + + // Test if the object is in stream + if (iter == m_Object.end ()) + return; + + if (iter->second.isDestroyed) + { + return; + } + + const ObjectInfo& info = iter->second; + + // Create empty object + Object* objectPtr = *outObjectPtr; + if (objectPtr == NULL) + { + *outObjectPtr = objectPtr = Object::Produce (info.classID, instanceId, kMemBaseObject, mode); + } + + if (objectPtr == NULL) + { + ErrorString ("Could not produce class with ID " + IntToString (info.classID)); + return; + } + SET_ALLOC_OWNER(this); +#if UNITY_EDITOR + (**outObjectPtr).SetFileIDHint (fileID); +#endif + + #if SUPPORT_SERIALIZED_TYPETREES + AssertIf (objectPtr->GetClassID () != info.classID); + bool perClassTypeTree = true; + Type* type = NULL; + if (!m_Type.empty() +#if SUPPORT_TEXT_SERIALIZATION + && !m_IsTextFile +#endif + ) + { + // Find TypeTree + type = &m_Type[info.typeID]; + AssertIf (type->GetOldType () == NULL); + + perClassTypeTree = !objectPtr->GetNeedsPerObjectTypeTree (); + + AssertIf (perClassTypeTree && objectPtr->GetClassID () != info.typeID); + // If we have a per class type tree we do not generate a typetree before reading the object + // That also means we always use SafeBinaryRead. + + // Setup new header type data + // If we have a perObject TypeTree we dont need the new type since we safebinaryread anyway. + // If we have a per class typetree we generate the typetree only once since it can't change + // while the application is running + if (type->GetNewType () == NULL && perClassTypeTree) + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + + int typeTreeOptions = kDontRequireAllMetaFlags | m_Options; + GenerateTypeTree (*objectPtr, typeTree, typeTreeOptions); + + // Register the type tree + type->SetNewType (typeTree); + + #if DEBUG_LOG_TYPETREE_MISMATCHES // Log when loaded typetree is out of sync. + if (!type->EqualTypes ()) + { + printf_console("Typetree mismatch in path: %s\n", m_DebugPath.c_str()); + printf_console("TYPETREE USED BY RUNTIME IS: \n"); + string buffer0; + type->GetOldType()->DebugPrint(buffer0); + printf_console(buffer0.c_str()); + + printf_console("TYPETREE IN FILE IS:\n"); + string buffer1; + type->GetNewType()->DebugPrint(buffer1); + printf_console(buffer1.c_str()); + } + #endif + } + AssertIf (type->EqualTypes () && !perClassTypeTree); + } + #endif + + int options = kNeedsInstanceIDRemapping | m_Options; + if (ShouldSwapEndian()) + options |= kSwapEndianess; + if (mode == kCreateObjectFromNonMainThread) + options |= kThreadedSerialization; + + objectPtr->SetIsPersistent(isPersistent); + + int byteStart = info.byteStart + m_ReadOffset; + + #if SUPPORT_SERIALIZED_TYPETREES + // Fill object with data + + // Never use the SafeBinaryRead code path for Meshes serialized with 3.5, because that needs a special workaround, + // which requires using StreamedBinaryRead. + if (type != NULL && !type->EqualTypes () && !((options & kWorkaround35MeshSerializationFuckup) && info.classID == ClassID(Mesh))) + { + SafeBinaryRead readStream; + + CachedReader& cache = readStream.Init (*type->GetOldType (), byteStart, info.byteSize , options); + cache.InitRead (*m_ReadFile, byteStart, info.byteSize); + Assert(m_ResourceImageGroup.resourceImages[0] == NULL); + + objectPtr->Reset (); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart > info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = perClassTypeTree; + } + else + #endif + { + // we will read up that object - no need to call Reset as we will construct it fully + objectPtr->HackSetResetWasCalled(); + + ///@TODO: Strip endianess! +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + YAMLRead readStream (m_ReadFile, byteStart, byteStart + info.byteSize, options, &m_DebugPath, info.debugLineStart); + objectPtr->VirtualRedirectTransfer (readStream); + *didChangeTypeTree = false; + } + else +#endif + if (!ShouldSwapEndian()) + { + StreamedBinaryRead readStream; + CachedReader& cache = readStream.Init (options); + cache.InitRead (*m_ReadFile, info.byteStart + m_ReadOffset, info.byteSize); + cache.InitResourceImages (m_ResourceImageGroup); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart != info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = false; + } + else + { + #if SUPPORT_SERIALIZED_TYPETREES + StreamedBinaryRead readStream; + CachedReader& cache = readStream.Init (options); + + cache.InitRead (*m_ReadFile, byteStart, info.byteSize); + Assert(m_ResourceImageGroup.resourceImages[0] == NULL); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart != info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = false; + #else + AssertString("reading endian swapped is not supported"); + #endif + } + } + + #if SUPPORT_SERIALIZED_TYPETREES + if (type) + *oldTypeTree = type->GetOldType (); + else + *oldTypeTree = NULL; + #endif + + // Setup hide flags when loading from a resource file + if (m_Options & kIsBuiltinResourcesFile) + objectPtr->SetHideFlagsObjectOnly(Object::kHideAndDontSave | Object::kHideInspector); + + return; +} + +#endif + +bool SerializedFile::IsEmpty () const +{ + for (ObjectMap::const_iterator i = m_Object.begin (); + i != m_Object.end (); ++i) + { + if (!i->second.isDestroyed) + return false; + } + return true; +} + +void SerializedFile::AddExternalRef (const FileIdentifier& pathName) +{ + // Dont' check for pathname here - it can be empty, if we are getting a GUID from + // a text serialized file, and the file belonging to the GUID is missing. In that + // case we just keep the GUID. + #if SUPPORT_SERIALIZE_WRITE + Assert (m_CachedWriter != NULL); + #endif + m_Externals.push_back (pathName); + m_Externals.back().CheckValidity(); +} + +void InitializeStdConverters () +{ + RegisterAllowTypeNameConversion ("PPtr", "PPtr"); + RegisterAllowTypeNameConversion ("PPtr", "PPtr"); + RegisterAllowTypeNameConversion ("PPtr", "PPtr"); + RegisterAllowTypeNameConversion ("PPtr", "PPtr"); + RegisterAllowTypeNameConversion ("PPtr", "PPtr"); + RegisterAllowTypeNameConversion ("PPtr