diff options
Diffstat (limited to 'Runtime/Serialize')
68 files changed, 15679 insertions, 0 deletions
diff --git a/Runtime/Serialize/AwakeFromLoadQueue.cpp b/Runtime/Serialize/AwakeFromLoadQueue.cpp new file mode 100644 index 0000000..fdb31de --- /dev/null +++ b/Runtime/Serialize/AwakeFromLoadQueue.cpp @@ -0,0 +1,395 @@ +#include "UnityPrefix.h" +#include "AwakeFromLoadQueue.h" +#include "Runtime/Profiler/Profiler.h" +#include "Runtime/Mono/MonoBehaviour.h" +#include "Runtime/Mono/MonoScript.h" +#include "Runtime/Misc/BuildSettings.h" +#if UNITY_EDITOR +#include "Editor/Src/Prefabs/PrefabBackwardsCompatibility.h" +#endif + +PROFILER_INFORMATION(gAwakeFromLoadQueue, "Loading.AwakeFromLoad", kProfilerLoading) + +AwakeFromLoadQueue::AwakeFromLoadQueue (MemLabelRef label) +{ + for(int i=0;i<kMaxQueues;++i) + m_ItemArrays[i].set_memory_label(label); +} + +void AwakeFromLoadQueue::Reserve (unsigned size) +{ + for (int i=0;i<kMaxQueues;i++) + { + if (i == kManagersQueue) + continue; + + m_ItemArrays[i].reserve(size); + } +} + +void AwakeFromLoadQueue::RegisterObjectInstanceIDs () +{ + LockObjectCreation(); + for (int i=0;i<kMaxQueues;i++) + RegisterObjectInstanceIDsInternal(m_ItemArrays[i].begin(), m_ItemArrays[i].size()); + UnlockObjectCreation(); +} + +void AwakeFromLoadQueue::RegisterObjectInstanceIDsInternal (Item* objects, unsigned size) +{ + for (int i=0;i<size;i++) + { + Object* ptr = objects[i].registerObjectPtr; + Assert(ptr != NULL); + Object::RegisterInstanceIDNoLock(ptr); + } +} + +int AwakeFromLoadQueue::DetermineQueueIndex(ClassIDType classID) +{ + if (classID == ClassID(MonoBehaviour)) + return kMonoBehaviourQueue; + else if (classID == ClassID(TerrainData)) + return kTerrainsQueue; + else if (classID == ClassID(Animator)) + return kAnimatorQueue; + else if (classID == ClassID(Rigidbody) +#if ENABLE_2D_PHYSICS + || classID == ClassID(Rigidbody2D) +#endif + ) + return kRigidbodyQueue; + else if (classID == ClassID(GameObject) || Object::IsDerivedFromClassID(classID, ClassID(Component))) + return kGameObjectAndComponentQueue; + else if (Object::IsDerivedFromClassID(classID, ClassID(GameManager))) + return kManagersQueue; + else + return kAssetQueue; + +} + +bool AwakeFromLoadQueue::IsInQueue(Object& target) +{ + int queueIndex = DetermineQueueIndex(target.GetClassID()); + + for (int i = 0; i < m_ItemArrays[queueIndex].size(); i++) + { + if (m_ItemArrays[queueIndex][i].objectPPtr == PPtr<Object>(target.GetInstanceID())) + return true; + } + + return false; +} + + +void AwakeFromLoadQueue::Add (Object& target, TypeTree* oldType, bool safeBinaryLoaded, AwakeFromLoadMode awakeOverride) +{ + Item item; + item.registerObjectPtr = ⌖ + item.objectPPtr = ⌖ + item.classID = target.GetClassID(); + #if UNITY_EDITOR + item.oldType = oldType; + item.safeBinaryLoaded = safeBinaryLoaded; + item.awakeModeOverride = awakeOverride; + #else + Assert(awakeOverride == -1); + #endif + + int queueIndex = DetermineQueueIndex(item.classID); + + m_ItemArrays[queueIndex].push_back(item); +} + +bool SortItemByInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return lhs.objectPPtr.GetInstanceID() < rhs.objectPPtr.GetInstanceID(); +} + +static int GetScriptExecutionOrder (int instanceID) +{ + #if ENABLE_SCRIPTING + MonoBehaviour* behaviour = dynamic_instanceID_cast<MonoBehaviour*> (instanceID); + if (behaviour != NULL) + { + MonoScript* script = behaviour->GetScript(); + if (script) + return script->GetExecutionOrder(); + } + #endif + return 0; +} + +bool AwakeFromLoadQueue::SortBehaviourByExecutionOrderAndInstanceID (int lhs, int rhs) +{ + int lhsExecutionOrder = GetScriptExecutionOrder(lhs); + int rhsExecutionOrder = GetScriptExecutionOrder(rhs); + + if (lhsExecutionOrder != rhsExecutionOrder) + return lhsExecutionOrder < rhsExecutionOrder; + else + return lhs < rhs; +} + +bool SortBehaviourByExecutionOrderAndReverseInstanceID (int lhs, int rhs) +{ + int lhsExecutionOrder = GetScriptExecutionOrder(lhs); + int rhsExecutionOrder = GetScriptExecutionOrder(rhs); + + if (lhsExecutionOrder != rhsExecutionOrder) + return lhsExecutionOrder < rhsExecutionOrder; + else + return lhs > rhs; +} + +static bool SortBehaviourItemByExecutionOrderAndInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return AwakeFromLoadQueue::SortBehaviourByExecutionOrderAndInstanceID(lhs.objectPPtr.GetInstanceID(), rhs.objectPPtr.GetInstanceID()); +} + +static bool SortBehaviourItemByExecutionOrderAndReverseInstanceID (const AwakeFromLoadQueue::Item& lhs, const AwakeFromLoadQueue::Item& rhs) +{ + return SortBehaviourByExecutionOrderAndReverseInstanceID(lhs.objectPPtr.GetInstanceID(), rhs.objectPPtr.GetInstanceID()); +} + +///@TODO: Should check consistency always be called immediately before calling Awake??? + +void AwakeFromLoadQueue::PersistentManagerAwakeFromLoad (AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + for (int i=0;i<kMaxQueues;i++) + PersistentManagerAwakeFromLoad(i, mode, safeBinaryCallback); +} + +void AwakeFromLoadQueue::ClearQueue (int queueIndex) +{ + m_ItemArrays[queueIndex].clear(); +} + + +void AwakeFromLoadQueue::Clear () +{ + for (int i=0;i<kMaxQueues;i++) + ClearQueue (i); +} + + +void AwakeFromLoadQueue::PersistentManagerAwakeFromLoad (int queueIndex, AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + Item* array = m_ItemArrays[queueIndex].begin(); + size_t size = m_ItemArrays[queueIndex].size(); + + std::sort(array, array + size, SortItemByInstanceID); + + if (queueIndex == kMonoBehaviourQueue) + { + if (UNITY_EDITOR) + { + // In the editor the execution order can be changed by the user at any time, + // thus we need to sort on load + std::sort(array, array + size, SortBehaviourItemByExecutionOrderAndInstanceID); + } + else + { + // In the player we write the scene files sorted by execution order + for (int j=1;j<size;j++) + { + AssertIf(SortBehaviourItemByExecutionOrderAndInstanceID(array[j], array[j-1])); + } + } + } + + InvokePersistentManagerAwake(array, size, mode, safeBinaryCallback); +} + +void AwakeFromLoadQueue::AwakeFromLoad (AwakeFromLoadMode mode) +{ + for (int i=0;i<kMaxQueues;i++) + { + // In 4.0 we started sorting prefab instantiation by script execution order and instanceID + if (IS_CONTENT_NEWER_OR_SAME (kUnityVersion4_0_a1)) + { + if (i == kMonoBehaviourQueue) + { + // For Instantiate we need sort by execution order + // The default order in Instantiate is determined by walking the hierarchy and components. + // Since instanceIDs are generated negative and decreasing we sort by reverse instanceID for components that do not. + std::sort(m_ItemArrays[i].begin(), m_ItemArrays[i].end(), SortBehaviourItemByExecutionOrderAndReverseInstanceID); + } + } + + InvokeAwakeFromLoad(m_ItemArrays[i].begin(), m_ItemArrays[i].size(), mode); + } +} + +void AwakeFromLoadQueue::CheckConsistency () +{ + for (int i=0;i<kMaxQueues;i++) + InvokeCheckConsistency(m_ItemArrays[i].begin(), m_ItemArrays[i].size()); +} + +#if UNITY_EDITOR + +void AwakeFromLoadQueue::InsertAwakeFromLoadQueue (ItemArray& src, ItemArray& dst, AwakeFromLoadMode awakeOverride) +{ + std::sort(src.begin(), src.end(), SortItemByInstanceID); + std::sort(dst.begin(), dst.end(), SortItemByInstanceID); + + // Inject any non-duplicate elements into the dst array + int oldDstSize = dst.size(); + int d = 0; + for (int i=0;i<src.size();i++) + { + while (d < oldDstSize && dst[d].objectPPtr.GetInstanceID() < src[i].objectPPtr.GetInstanceID()) + d++; + + if (d >= oldDstSize || dst[d].objectPPtr.GetInstanceID() != src[i].objectPPtr.GetInstanceID()) + { + dst.push_back(src[i]); + } + // else -> The object is in the destination array already. + } +} + +void AwakeFromLoadQueue::InsertAwakeFromLoadQueue (AwakeFromLoadQueue& awakeFromLoadQueue, AwakeFromLoadMode awakeOverride) +{ + for (int i=0;i<kMaxQueues;i++) + InsertAwakeFromLoadQueue(awakeFromLoadQueue.m_ItemArrays[i], m_ItemArrays[i], awakeOverride); +} + +void AwakeFromLoadQueue::PatchPrefabBackwardsCompatibility () +{ + for (int q=0;q<kMaxQueues;q++) + { + int size = m_ItemArrays[q].size(); + Item* objects = m_ItemArrays[q].begin(); + + for (int i=0;i<size;i++) + { + Object* ptr = objects[i].objectPPtr; + if (ptr) + { + TypeTree* typeTree = objects[i].oldType; + if (typeTree != NULL && objects[i].safeBinaryLoaded) + { + objects[i].safeBinaryLoaded = false; + RemapOldPrefabOverrideFromLoading (*ptr, *typeTree); + } + + Prefab* prefab = dynamic_pptr_cast<Prefab*> (ptr); + if (prefab) + prefab->PatchPrefabBackwardsCompatibility(); + + EditorExtension* extension = dynamic_pptr_cast<EditorExtension*> (ptr); + if (extension) + extension->PatchPrefabBackwardsCompatibility(); + } + } + } +} +#endif + + +void AwakeFromLoadQueue::InvokePersistentManagerAwake (Item* objects, unsigned size, AwakeFromLoadMode awakeMode, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + #if DEBUGMODE + int previousInstanceID = 0; + #endif + + for (int i=0;i<size;i++) + { + PROFILER_AUTO(gAwakeFromLoadQueue, NULL) + + // The AwakeFromLoadQueue should never have any duplicate elements. + #if DEBUGMODE + Assert(objects[i].objectPPtr.GetInstanceID() != previousInstanceID); + previousInstanceID = objects[i].objectPPtr.GetInstanceID(); + #endif + + Object* ptr = objects[i].objectPPtr; + if (ptr == NULL) + continue; + + #if UNITY_EDITOR + // Loaded with SafeBinaryRead thus needs the callback + if (objects[i].safeBinaryLoaded && safeBinaryCallback != NULL) + safeBinaryCallback (*ptr, *objects[i].oldType); + ptr->CheckConsistency (); + + #endif + + AwakeFromLoadMode objectAwakeMode = awakeMode; + #if UNITY_EDITOR + if (objects[i].awakeModeOverride != kDefaultAwakeFromLoadInvalid) + objectAwakeMode = objects[i].awakeModeOverride; + #endif + + ptr->AwakeFromLoad (objectAwakeMode); + ptr->ClearPersistentDirty (); + } +} + +void AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (Object& o, TypeTree* oldType, AwakeFromLoadMode awakeMode, bool safeBinaryLoaded, SafeBinaryReadCallbackFunction* safeBinaryCallback) +{ + PROFILER_AUTO(gAwakeFromLoadQueue, &o) + + #if UNITY_EDITOR + // Loaded with SafeBinaryRead thus needs the callback + if (safeBinaryLoaded && safeBinaryCallback != NULL) + safeBinaryCallback (o, *oldType); + o.CheckConsistency (); + #endif + + o.AwakeFromLoad (awakeMode); + o.ClearPersistentDirty (); +} + +void AwakeFromLoadQueue::InvokeAwakeFromLoad (Item* objects, unsigned size, AwakeFromLoadMode mode) +{ + for (int i=0;i<size;i++) + { + Object* ptr = objects[i].objectPPtr; + if (ptr) + { + #if UNITY_EDITOR + if (objects[i].awakeModeOverride != kDefaultAwakeFromLoadInvalid) + { + ptr->AwakeFromLoad(objects[i].awakeModeOverride); + continue; + } + + #endif + + + ptr->AwakeFromLoad(mode); + } + } +} + +void AwakeFromLoadQueue::InvokeCheckConsistency (Item* objects, unsigned size) +{ + for (int i=0;i<size;i++) + { + Object* ptr = objects[i].objectPPtr; + if (ptr) + ptr->CheckConsistency(); + } +} + +void AwakeFromLoadQueue::ExtractAllObjects (dynamic_array<PPtr<Object> >& outObjects) +{ + Assert(outObjects.empty()); + + int count = 0; + for (int q=0;q<kMaxQueues;q++) + count += m_ItemArrays[q].size(); + outObjects.reserve(count); + + for (int q=0;q<kMaxQueues;q++) + { + int size = m_ItemArrays[q].size(); + Item* objects = m_ItemArrays[q].begin(); + + for (int i=0;i<size;i++) + outObjects.push_back(objects[i].objectPPtr); + } +} diff --git a/Runtime/Serialize/AwakeFromLoadQueue.h b/Runtime/Serialize/AwakeFromLoadQueue.h new file mode 100644 index 0000000..cc8fbe2 --- /dev/null +++ b/Runtime/Serialize/AwakeFromLoadQueue.h @@ -0,0 +1,83 @@ +#ifndef AWAKE_FROM_LOAD_QUEUE_H +#define AWAKE_FROM_LOAD_QUEUE_H + +#include "Runtime/BaseClasses/BaseObject.h" +#include "Runtime/Utilities/dynamic_array.h" +class TypeTree; + + +// Rigidbodies must run before Colliders +// Animators must come after all other components because they bind properties (eg. SkinnedMeshRenderer needs to be fully initialized to bind blendshapes) +// Terrains depend on prefabs for trees and textures to be fully prepared. +// MonoBehaviours always come last since they can call anything on any component or asset. + +enum { kManagersQueue = 0, kAssetQueue, kRigidbodyQueue, kGameObjectAndComponentQueue, kAnimatorQueue, kTerrainsQueue, kMonoBehaviourQueue, kMaxQueues }; + +class AwakeFromLoadQueue +{ +public: + + struct Item + { + Object* registerObjectPtr; + + PPtr<Object> objectPPtr; + ClassIDType classID; + +#if UNITY_EDITOR + TypeTree* oldType; + bool safeBinaryLoaded; + AwakeFromLoadMode awakeModeOverride; +#endif + }; + + AwakeFromLoadQueue (MemLabelRef label); + + typedef dynamic_array<Item> ItemArray; + + typedef void SafeBinaryReadCallbackFunction (Object& object, const TypeTree& oldTypeTree); + + bool IsInQueue(Object& target); + + void Reserve (unsigned size); + void Add (Object& target, TypeTree* oldType = NULL, bool safeBinaryLoaded = false, AwakeFromLoadMode awakeOverride = kDefaultAwakeFromLoadInvalid); + + static void PersistentManagerAwakeSingleObject (Object& o, TypeTree* oldType, AwakeFromLoadMode awakeMode, bool safeBinaryLoaded, SafeBinaryReadCallbackFunction* safeBinaryCallback); + + void PersistentManagerAwakeFromLoad (AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + void PersistentManagerAwakeFromLoad (int queueIndex, AwakeFromLoadMode mode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + + void ClearQueue (int queueIndex); + void Clear (); + + void AwakeFromLoad (AwakeFromLoadMode mode); + void CheckConsistency (); + + void RegisterObjectInstanceIDs (); + + #if UNITY_EDITOR + void PatchPrefabBackwardsCompatibility (); + void InsertAwakeFromLoadQueue (AwakeFromLoadQueue& awakeFromLoadQueue, AwakeFromLoadMode awakeOverride); + #endif + + void ExtractAllObjects (dynamic_array<PPtr<Object> >& outObjects); + + ItemArray& GetItemArray (int queueIndex) { return m_ItemArrays[queueIndex]; } + + static bool SortBehaviourByExecutionOrderAndInstanceID (int lhs, int rhs); + +private: + int DetermineQueueIndex(ClassIDType classID); + + static void InvokeAwakeFromLoad (Item* objects, unsigned size, AwakeFromLoadMode mode); + static void InvokeCheckConsistency (Item* objects, unsigned size); + static void InvokePersistentManagerAwake (Item* objects, unsigned size, AwakeFromLoadMode awakeMode, SafeBinaryReadCallbackFunction* safeBinaryCallback); + static void RegisterObjectInstanceIDsInternal (Item* objects, unsigned size); + + void InsertAwakeFromLoadQueue (dynamic_array<Item>& src, dynamic_array<Item>& dst, AwakeFromLoadMode awakeOverride); + + + ItemArray m_ItemArrays[kMaxQueues]; +}; + +#endif
\ No newline at end of file diff --git a/Runtime/Serialize/Blobification/BlobSize.h b/Runtime/Serialize/Blobification/BlobSize.h new file mode 100644 index 0000000..a7aaf6d --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobSize.h @@ -0,0 +1,161 @@ +#ifndef BLOBSIZE_H +#define BLOBSIZE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" + +struct ReduceCopyData; + +class BlobSize : public TransferBase +{ +private: + + typedef OffsetPtr<void*>::offset_type offset_type; + + size_t m_Size; + bool m_IgnorePtr; + bool m_HasDebugOffsetPtr; + bool m_Use64Ptr; + int m_TargetPlatform; + + std::size_t AlignAddress(std::size_t addr, std::size_t align) + { + return addr + ((~addr + 1U) & (align - 1U)); + } + +public: + + + BlobSize (bool hasDebugOffsetPtr, bool use64BitOffsetPtr) + : m_Size (0) + , m_IgnorePtr (false) + , m_HasDebugOffsetPtr (hasDebugOffsetPtr) + , m_Use64Ptr (use64BitOffsetPtr) { } + + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + int GetFlags () { return m_Flags; } + bool IsWritingGameReleaseData () + { + return true; + } + bool IsSerializingForGameRelease () + { + return true; + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + if (platform == kBuildAnyPlayerData) + return m_TargetPlatform >= kBuildValidPlayer; + else + return m_TargetPlatform == platform; + } + + BuildTargetPlatform GetBuildingTargetPlatform () { return static_cast<BuildTargetPlatform>(m_TargetPlatform); } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData); + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + static size_t CalculateSize (T& data, bool hasDebugOffsetPtr, bool use64BitOffsetPtr) + { + BlobSize sizer (hasDebugOffsetPtr, use64BitOffsetPtr); + sizer.Transfer(data, "Base"); + + // size_t size = sizeof(T); + // Assert( sizeof(T) == sizer.m_Size); + + return sizer.m_Size; + } + + template<class> friend class BlobSizeTransferSTLStyleArrayImpl; +}; + + +template<typename T> class BlobSizeTransferSTLStyleArrayImpl +{ +public: + void operator()(T& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + AssertString ("STL array are not support for BlobWrite"); + } +}; + +template<typename T> class BlobSizeTransferSTLStyleArrayImpl< OffsetPtrArrayTransfer<T> > +{ +public: + void operator()(OffsetPtrArrayTransfer<T>& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + transfer.m_IgnorePtr = false; + } +}; + +template<typename T, int SIZE> class BlobSizeTransferSTLStyleArrayImpl< StaticArrayTransfer<T, SIZE> > +{ +public: + void operator()(StaticArrayTransfer<T, SIZE>& data, TransferMetaFlags metaFlags, BlobSize& transfer) + { + transfer.m_Size = transfer.AlignAddress(transfer.m_Size, ALIGN_OF(T)); + transfer.m_Size += sizeof(T)*data.size(); + } +}; + + +template<class T> inline +void BlobSize::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlags) +{ + BlobSizeTransferSTLStyleArrayImpl<T> transfer; + transfer(data, metaFlags, *this); +} + +template<class T> inline +void BlobSize::Transfer (T& data, const char* name, TransferMetaFlags) +{ + if (m_IgnorePtr) + { + m_IgnorePtr = false; + return; + } + + m_Size = AlignAddress(m_Size, SerializeTraits<T>::GetAlignOf() ); + + SerializeTraits<T>::Transfer (data, *this); + + m_Size = AlignAddress(m_Size, SerializeTraits<T>::GetAlignOf() ); +} + +template<class T> inline +void BlobSize::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<class T> inline +void BlobSize::TransferBasicData (T& srcData) +{ + m_Size += sizeof(T); +} + +template<class T> inline +void BlobSize::TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData) +{ + m_Size += m_Use64Ptr ? sizeof(SInt64) : sizeof(SInt32); + + if (m_HasDebugOffsetPtr) + m_Size += sizeof(void*); + + if (isValidPtr) + m_IgnorePtr = true; +} + +#endif diff --git a/Runtime/Serialize/Blobification/BlobTests.cpp b/Runtime/Serialize/Blobification/BlobTests.cpp new file mode 100644 index 0000000..8b0a2d3 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobTests.cpp @@ -0,0 +1,358 @@ +#include "UnityPrefix.h" + +#if ENABLE_UNIT_TESTS && !(UNITY_LINUX && UNITY_64) +#include "External/UnitTest++/src/UnitTest++.h" +#include <limits> +#include "Runtime/Math/Vector3.h" +#include "Runtime/Math/Simd/math.h" +#include "Runtime/mecanim/defs.h" +#include "Runtime/mecanim/types.h" +#include "Runtime/Allocator/MemoryManager.h" +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/Blobification/offsetptr.h" +#include "Runtime/Serialize/Blobification/BlobWrite.h" + +template<typename TYPE> TYPE* Construct() +{ + #if ENABLE_MEMORY_MANAGER + void* _where = GetMemoryManager().Allocate (sizeof(TYPE), ALIGN_OF(TYPE), MemLabelId(kMemNewDeleteId, NULL), kAllocateOptionNone, "Construct"); + #else + void* _where = malloc(sizeof(TYPE)); + #endif + return new(_where) TYPE; +} + +template<typename TYPE> TYPE* ConstructArray(size_t num) +{ + #if ENABLE_MEMORY_MANAGER + void* _where = GetMemoryManager().Allocate (sizeof(TYPE)*num, ALIGN_OF(TYPE), MemLabelId(kMemNewDeleteId, NULL), kAllocateOptionNone, "ConstructArray"); + #else + void* _where = malloc(sizeof(TYPE)*num); + #endif + return new(_where) TYPE[num]; +} + +template<typename TYPE> void Destruct(TYPE* p) +{ + #if ENABLE_MEMORY_MANAGER + GetMemoryManager().Deallocate (p, MemLabelId(kMemNewDeleteId, NULL)); + #else + free(p); + #endif +} + + +struct SampleDataA +{ + enum { + kLastIndex = 20 + }; + + int intValue1; + math::float4 float4Value; // (intentially unaligned) + Vector3f vector3; + + mecanim::uint32_t index[kLastIndex]; + + OffsetPtr<float> nullPtr; + + OffsetPtr<float> floatPtr; + + mecanim::uint32_t arraySize; + OffsetPtr<float> array; + + mecanim::uint32_t emptyArraySize; + OffsetPtr<math::float4> emptyArray; + + int intValue2; + + DECLARE_SERIALIZE(SampleData) +}; + +struct SampleData +{ + int intValue1; // 0 + math::float4 float4Value; // 16 (intentially unaligned) + Vector3f vector3; // 32 + + OffsetPtr<float> nullPtr; // 44 + + OffsetPtr<float> floatPtr; // 52 + + mecanim::uint32_t arraySize; // 60 + OffsetPtr<double> array; // 64 + + mecanim::uint32_t emptyArraySize; // 72 + OffsetPtr<math::float4> emptyArray; // 76 + + mecanim::uint32_t sampleDataASize; // 84 + OffsetPtr<SampleDataA> sampleDataA; // 88 + + mecanim::uint32_t sampleDataAHandleSize; // 96 + OffsetPtr<OffsetPtr<SampleDataA> > sampleDataAHandle; // 100 + + int intValue2; // 108 + + DECLARE_SERIALIZE(SampleData) +}; + + + +template<class TransferFunction> inline +void SampleDataA::Transfer(TransferFunction& transfer) +{ + TRANSFER(intValue1); + TRANSFER(float4Value); + TRANSFER(vector3); + + STATIC_ARRAY_TRANSFER(mecanim::uint32_t, index, kLastIndex); + + TRANSFER(nullPtr); + TRANSFER(floatPtr); + + TRANSFER_BLOB_ONLY(arraySize); + MANUAL_ARRAY_TRANSFER2(float, array, arraySize); + + + TRANSFER_BLOB_ONLY(emptyArraySize); + MANUAL_ARRAY_TRANSFER2(math::float4, emptyArray, emptyArraySize); + + TRANSFER(intValue2); +} + + +template<class TransferFunction> inline +void SampleData::Transfer(TransferFunction& transfer) +{ + TRANSFER(intValue1); + TRANSFER(float4Value); + TRANSFER(vector3); + + TRANSFER(nullPtr); + TRANSFER(floatPtr); + + TRANSFER_BLOB_ONLY(arraySize); + MANUAL_ARRAY_TRANSFER2(double, array, arraySize); + + + TRANSFER_BLOB_ONLY(emptyArraySize); + MANUAL_ARRAY_TRANSFER2(math::float4, emptyArray, emptyArraySize); + + TRANSFER_BLOB_ONLY(sampleDataASize); + MANUAL_ARRAY_TRANSFER2(SampleDataA, sampleDataA, sampleDataASize); + + TRANSFER_BLOB_ONLY(sampleDataAHandleSize); + MANUAL_ARRAY_TRANSFER2(OffsetPtr<SampleDataA>, sampleDataAHandle, sampleDataAHandleSize); + + TRANSFER(intValue2); +} + +static void SetupTestDataA (SampleDataA& sourceData) +{ + sourceData.intValue1 = 1; + sourceData.float4Value = math::float4(1, 2, 3, 4); + sourceData.vector3 = Vector3f(1,2,3); + + mecanim::uint32_t i; + for(i=0; i<SampleDataA::kLastIndex;i++) + sourceData.index[i] = i; + + sourceData.nullPtr = NULL; + sourceData.floatPtr = new float; + *sourceData.floatPtr = 5.5F; + + sourceData.emptyArraySize = 0; + sourceData.emptyArray = NULL; + + sourceData.arraySize = 3; + sourceData.array = new float[3]; + sourceData.array[0] = 6.5f; + sourceData.array[1] = 7.5f; + sourceData.array[2] = 8.5f; + sourceData.intValue2 = 2; +} + +static void DeleteTestDataA (SampleDataA& sourceData) +{ + delete[] sourceData.array.Get(); + delete sourceData.floatPtr.Get(); +} + + +static void SetupTestData (SampleData& sourceData) +{ + sourceData.intValue1 = 1; + sourceData.float4Value = math::float4(1, 2, 3, 4); + sourceData.vector3 = Vector3f(1,2,3); + sourceData.nullPtr = NULL; + sourceData.floatPtr = new float; + *sourceData.floatPtr = 5.5F; + + sourceData.emptyArraySize = 0; + sourceData.emptyArray = NULL; + + sourceData.arraySize = 3; + sourceData.array = new double[3]; + sourceData.array[0] = 6.5; + sourceData.array[1] = 7.5; + sourceData.array[2] = 8.5; + sourceData.intValue2 = 2; + + sourceData.sampleDataASize = 4; + + sourceData.sampleDataA = ConstructArray<SampleDataA>(sourceData.sampleDataASize); + for(int i=0;i<sourceData.sampleDataASize;i++) + { + SetupTestDataA(sourceData.sampleDataA[i]); + } + + sourceData.sampleDataAHandleSize = 2; + sourceData.sampleDataAHandle = new OffsetPtr<SampleDataA> [2]; + sourceData.sampleDataAHandle[0] = Construct<SampleDataA>(); + SetupTestDataA(*sourceData.sampleDataAHandle[0]); + sourceData.sampleDataAHandle[1] = Construct<SampleDataA>(); + SetupTestDataA(*sourceData.sampleDataAHandle[1]); +} + + +static void DeleteTestData (SampleData& sourceData) +{ + for(int i=0;i<sourceData.sampleDataASize;i++) + DeleteTestDataA (sourceData.sampleDataA[i]); + + Destruct (sourceData.sampleDataA.Get()); + Destruct (sourceData.sampleDataAHandle[0].Get()); + Destruct (sourceData.sampleDataAHandle[1].Get()); + + delete sourceData.floatPtr.Get(); + delete[] sourceData.array.Get(); + delete[] sourceData.sampleDataAHandle.Get(); +} + + +static void TestDataA (SampleDataA& deserialized) +{ + CHECK (reinterpret_cast<UInt32>(&deserialized) % ALIGN_OF(SampleDataA) == 0 ); + CHECK (deserialized.intValue1 == 1); + CHECK (dot(deserialized.float4Value - math::float4(1, 2, 3, 4)) == math::float1::zero()); + + mecanim::uint32_t i; + for(i=0; i<SampleDataA::kLastIndex;i++) + CHECK (deserialized.index[i] == i); + + CHECK (deserialized.nullPtr.IsNull()); + + float ptr = *deserialized.floatPtr; + CHECK (ptr == 5.5F); + + float* array = deserialized.array.Get(); + CHECK (array[0] == 6.5F); + CHECK (array[1] == 7.5F); + CHECK (array[2] == 8.5F); + + CHECK (deserialized.emptyArray.IsNull()); + CHECK (deserialized.emptyArraySize == 0); + + + CHECK (deserialized.vector3 == Vector3f(1,2,3)); + CHECK (deserialized.intValue2 == 2); +} + + +static void TestData (SampleData& deserialized) +{ + CHECK (reinterpret_cast<UInt32>(&deserialized) % ALIGN_OF(SampleData) == 0); + CHECK (deserialized.intValue1 == 1); + CHECK (dot(deserialized.float4Value - math::float4(1, 2, 3, 4)) == math::float1::zero() ); + + CHECK (deserialized.nullPtr.IsNull()); + + float ptr = *deserialized.floatPtr; + CHECK (ptr == 5.5F); + + double* array = deserialized.array.Get(); + CHECK (array[0] == 6.5); + CHECK (array[1] == 7.5); + CHECK (array[2] == 8.5); + + CHECK (deserialized.emptyArray.IsNull()); + CHECK (deserialized.emptyArraySize == 0); + + CHECK (deserialized.vector3 == Vector3f(1,2,3)); + CHECK (deserialized.intValue2 == 2); + + CHECK (deserialized.sampleDataASize == 4); + for(int i=0;i<deserialized.sampleDataASize;i++) + { + TestDataA(deserialized.sampleDataA[i]); + } + + CHECK (deserialized.sampleDataAHandleSize == 2); + for(int i=0;i<deserialized.sampleDataAHandleSize;i++) + { + TestDataA(*deserialized.sampleDataAHandle[i]); + } +} + +// This test crashes the linux-amd64 editor right now +SUITE (BlobTests) +{ + TEST (Blobification_BlobPtrs) + { + SampleData sourceData; + SetupTestData (sourceData); + TestData(sourceData); + + // Generate blob + BlobWrite::container_type data; + BlobWrite blobWrite (data, kNoTransferInstructionFlags, kBuildNoTargetPlatform); + blobWrite.Transfer(sourceData, "Base"); + TestData(*reinterpret_cast<SampleData*> (data.begin())); + + // Generate blob with reduce copy + BlobWrite::container_type dataReduced; + BlobWrite blobWriteReduce (dataReduced, kNoTransferInstructionFlags, kBuildNoTargetPlatform); + blobWriteReduce.SetReduceCopy(true); + blobWriteReduce.Transfer(sourceData, "Base"); + TestData(*reinterpret_cast<SampleData*> (dataReduced.begin())); + + // Ensure reduced blob is actually smaller. + CHECK (dataReduced.size() < data.size()); + + // Ensure that 64 bit data is larger than non-64 bit data + BlobWrite::container_type data64; + BlobWrite blobWrite64 (data64, kNoTransferInstructionFlags, kBuildStandaloneWin64Player); + blobWrite64.Transfer(sourceData, "Base"); + + BlobWrite::container_type data32; + BlobWrite blobWrite32 (data32, kNoTransferInstructionFlags, kBuildStandaloneWinPlayer); + blobWrite32.Transfer(sourceData, "Base"); + CHECK (data64.size() > data32.size()); + + DeleteTestData (sourceData); + } + + TEST (Blobification_OffsetPtr) + { + OffsetPtr<size_t>* ptrHigh = new OffsetPtr<size_t>; + OffsetPtr<size_t>* ptrLow = new OffsetPtr<size_t>; + + size_t* ptrH = reinterpret_cast<size_t*>(std::numeric_limits<size_t>::max()-4); + size_t* ptrL = reinterpret_cast<size_t*>(4); + + ptrHigh->reset(ptrH); + ptrLow->reset(ptrL); + + size_t h = reinterpret_cast<size_t>(ptrHigh->Get()); + size_t l = reinterpret_cast<size_t>(ptrLow->Get()); + + + CHECK (h == std::numeric_limits<size_t>::max()-4); + CHECK (l == 4); + + delete ptrHigh; + delete ptrLow; + } +} + +#endif //ENABLE_UNIT_TESTS diff --git a/Runtime/Serialize/Blobification/BlobWrite.cpp b/Runtime/Serialize/Blobification/BlobWrite.cpp new file mode 100644 index 0000000..dcb7705 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWrite.cpp @@ -0,0 +1,148 @@ +#include "UnityPrefix.h" +#include "BlobWrite.h" +#include "Configuration/UnityConfigure.h" +#include "BlobWriteTargetSupport.h" + +BlobWrite::BlobWrite (container_type& blob, TransferInstructionFlags flags, BuildTargetPlatform targetPlatform) +: m_Blob(blob), + m_CopyData(true), + m_ReduceCopy(false), + m_TargetPlatform(targetPlatform) +{ + m_Flags = false; + m_SwapEndianess = m_Flags & kSwapEndianess; + m_Use64BitOffsetPtr = IsBuildTarget64BitBlob (targetPlatform); +} + +void BlobWrite::Push (size_t size, void* srcDataPtr, size_t align) +{ + Assert (m_CopyData); + + size_t offset = AlignAddress(m_Blob.size(), align); + m_Context.push( TypeContext(offset, 0, reinterpret_cast<UInt8*> (srcDataPtr), size) ); + m_Blob.resize_initialized(offset + size, 0); + m_CopyData = false; +} + +void BlobWrite::WritePtrValueAtLocation (size_t locationInBlob, SInt64 value) +{ + if (m_Use64BitOffsetPtr) + { + SInt64 offset64 = value; + + if (m_SwapEndianess) + SwapEndianBytes(offset64); + memcpy (&m_Blob[locationInBlob], &offset64, sizeof(offset64)); + } + else + { + SInt32 offset32 = value; + + if (m_SwapEndianess) + SwapEndianBytes(offset32); + memcpy (&m_Blob[locationInBlob], &offset32, sizeof(offset32)); + } +} + + +void BlobWrite::TransferPtrImpl (bool isValidPtr, ReduceCopyData* reduceCopyData, size_t alignOfT) +{ + Assert(!m_CopyData); + // When the data is null we will not call Transfer. + m_CopyData = isValidPtr; + + // Need to update OffsetPtr's member 'mOffset' + // compute member offset in memory buffer + size_t dataPosition = AlignAddress(m_Blob.size(), alignOfT); + size_t offset = GetActiveOffset(); + offset = dataPosition - offset; + if (!isValidPtr) + offset = 0; + + // Write the ptr + WritePtrValueAtLocation(GetActiveOffset (), offset); + + // Setup reduce copy data for later use by ReduceCopyImpl + if (reduceCopyData != NULL) + { + if (isValidPtr) + { + reduceCopyData->ptrPosition = GetActiveOffset(); + reduceCopyData->dataStart = dataPosition; + reduceCopyData->blobSize = m_Blob.size(); + } + else + { + reduceCopyData->ptrPosition = 0xFFFFF; + reduceCopyData->dataStart = 0xFFFFF; + reduceCopyData->blobSize = 0xFFFFF; + } + } + + + // Offset write location in the blob + m_Context.top().m_Offset += m_Use64BitOffsetPtr ? sizeof(SInt64) : sizeof(SInt32); + if (HasOffsetPtrWithDebugPtr()) + m_Context.top().m_Offset += sizeof(void*); +} + +bool BlobWrite::HasOffsetPtrWithDebugPtr () const +{ + return m_TargetPlatform == kBuildNoTargetPlatform; +} + +bool BlobWrite::AllowDataLayoutValidation () const +{ + size_t targetOffsetPtrSize = Use64BitOffsetPtr () ? sizeof(SInt64) : sizeof(SInt32); + if (HasOffsetPtrWithDebugPtr ()) + targetOffsetPtrSize += sizeof(void*); + + size_t srcOffsetPtrSize = sizeof(OffsetPtr<UInt8>); + + return targetOffsetPtrSize == srcOffsetPtrSize; +} + +// Ensure that the user has matching Transfer calls & Data layout in the struct +void BlobWrite::ValidateSerializedLayout (void* srcData, const char* name) +{ + UInt8* srcDataPtr = reinterpret_cast<UInt8*> (srcData); + + // (float4 and some others transfer functions, transfer temporary data, we ignore layout checks on those and hope for the best) + int srcDataOffset = srcDataPtr - m_Context.top().m_SourceDataPtr; + if (srcDataOffset < 0 || srcDataOffset >= m_Context.top().m_SourceDataSize) + return; + + // When targeting a platform with a different layout than our own, obviously these checks dont make sense... + if (!AllowDataLayoutValidation ()) + return; + + int blobOffset = m_Context.top().m_Offset; + if (srcDataOffset != blobOffset) + { + AssertString(Format("BlobWrite: Transfer '%s' is not called in the same order as the struct is laid out. Expected: %d got: %d ", name, srcDataOffset, blobOffset)); + } +} + +void BlobWrite::ReduceCopyImpl (const ReduceCopyData& reduce, size_t alignOfT) +{ + if (!m_ReduceCopy || reduce.dataStart == 0xFFFFF) + return; + + // Find any data in the blob that matches the last written data. + // if we find it, delete it again and reference the previous memory block instead + size_t dataSize = m_Blob.size() - reduce.dataStart; + for (int i=0;i < reduce.dataStart;i+=alignOfT) + { + if (memcmp(&m_Blob[i], &m_Blob[reduce.dataStart], dataSize) == 0) + { + // Update offset pointer + SInt64 offset = i - reduce.ptrPosition; + WritePtrValueAtLocation (reduce.ptrPosition, offset); + + // resize blob based on the reduce copy + m_Blob.resize_initialized(reduce.blobSize, 0); + + return; + } + } +} diff --git a/Runtime/Serialize/Blobification/BlobWrite.h b/Runtime/Serialize/Blobification/BlobWrite.h new file mode 100644 index 0000000..56a0340 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWrite.h @@ -0,0 +1,220 @@ +#ifndef BLOBWRITE_H +#define BLOBWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/SwapEndianBytes.h" +#include "Runtime/Animation/MecanimArraySerialization.h" +#include "offsetptr.h" +#include "BlobSize.h" +#include "ReduceCopyData.h" +#include <stack> +#include "Runtime/Modules/ExportModules.h" + +class EXPORT_COREMODULE BlobWrite : public TransferBase +{ +public: + + typedef dynamic_array<UInt8, 16> container_type; + +private: + + container_type& m_Blob; + int m_TargetPlatform; + bool m_CopyData; + bool m_ReduceCopy; + bool m_Use64BitOffsetPtr; + bool m_SwapEndianess; + + struct TypeContext + { + TypeContext(size_t root, size_t offset, UInt8* srcDataPtr,size_t srcDataSize):m_Root(root),m_Offset(offset),m_SourceDataPtr(srcDataPtr),m_SourceDataSize(srcDataSize) {} + + size_t m_Root; + size_t m_Offset; + + UInt8* m_SourceDataPtr; + size_t m_SourceDataSize; + }; + std::stack<TypeContext> m_Context; + + std::size_t AlignAddress(std::size_t addr, std::size_t align) + { + return addr + ((~addr + 1U) & (align - 1U)); + } + + size_t GetActiveOffset () const + { + return m_Context.top().m_Root + m_Context.top().m_Offset; + } + + UInt8* GetActiveBlobPtr () + { + return &m_Blob[GetActiveOffset ()]; + } + + void WritePtrValueAtLocation (size_t locationInBlob, SInt64 value); + + void ValidateSerializedLayout (void* srcData, const char* name); + + void Push (size_t size, void* srcDataPtr, size_t align); + + void TransferPtrImpl (bool isValidPtr, ReduceCopyData* reduceCopyData, size_t alignOfT); + void ReduceCopyImpl (const ReduceCopyData& reduce, size_t alignOfT); + + bool HasOffsetPtrWithDebugPtr () const; + bool Use64BitOffsetPtr() const { return m_Use64BitOffsetPtr; } + bool AllowDataLayoutValidation () const; + +public: + + BlobWrite (container_type& blob, TransferInstructionFlags flags, BuildTargetPlatform targetPlatform); + + void SetReduceCopy (bool reduce) { m_ReduceCopy = reduce; } + + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return m_SwapEndianess; } + bool IsWritingGameReleaseData () + { + return true; + } + bool IsSerializingForGameRelease () + { + return true; + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + if (platform == kBuildAnyPlayerData) + return m_TargetPlatform >= kBuildValidPlayer; + else + return m_TargetPlatform == platform; + } + + BuildTargetPlatform GetBuildingTargetPlatform () { return static_cast<BuildTargetPlatform>(m_TargetPlatform); } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData); + + template<class T> + void ReduceCopy (const ReduceCopyData& reduce); + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class> friend class BlobWriteTransferSTLStyleArrayImpl; +}; + +template<typename T> class BlobWriteTransferSTLStyleArrayImpl +{ +public: + void operator()(T& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + AssertString ("STL array are not support for BlobWrite"); + } +}; + +template<typename T> class BlobWriteTransferSTLStyleArrayImpl< OffsetPtrArrayTransfer<T> > +{ +public: + void operator()(OffsetPtrArrayTransfer<T>& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + if (data.size() == 0) + { + Assert (!transfer.m_CopyData); + return; + } + Assert (transfer.m_CopyData); + + size_t arrayByteSize = BlobSize::CalculateSize(*data.begin(), transfer.HasOffsetPtrWithDebugPtr(), transfer.Use64BitOffsetPtr()) * data.size(); + transfer.Push(arrayByteSize, &*data.begin(), ALIGN_OF( typename OffsetPtrArrayTransfer<T>::value_type )); + + typename OffsetPtrArrayTransfer<T>::iterator end = data.end (); + for (typename OffsetPtrArrayTransfer<T>::iterator i = data.begin ();i != end;++i) + transfer.Transfer (*i, "data"); + + transfer.m_Context.pop(); + } +}; + +template<typename T, int SIZE> class BlobWriteTransferSTLStyleArrayImpl< StaticArrayTransfer<T, SIZE> > +{ +public: + void operator()(StaticArrayTransfer<T, SIZE>& data, TransferMetaFlags metaFlags, BlobWrite& transfer) + { + typename StaticArrayTransfer<T, SIZE>::iterator end = data.end (); + for (typename StaticArrayTransfer<T, SIZE>::iterator i = data.begin ();i != end;++i) + transfer.Transfer (*i, "data"); + } +}; + +template<class T> inline +void BlobWrite::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlags) +{ + BlobWriteTransferSTLStyleArrayImpl<T> transfer; + transfer(data, metaFlags, *this); +} + +template<class T> inline +void BlobWrite::Transfer (T& data, const char* name, TransferMetaFlags) +{ + bool copyData = m_CopyData; + if (m_CopyData) + Push(BlobSize::CalculateSize(data, HasOffsetPtrWithDebugPtr(), Use64BitOffsetPtr()), &data, SerializeTraits<T>::GetAlignOf() ); + + // Follow natural c++ alignment + size_t head = m_Context.top().m_Root; + size_t& offset = m_Context.top().m_Offset; + // always align head + offset not only offset otherwise you may get wrong align for nested data structure + offset = AlignAddress(head + offset, SerializeTraits<T>::GetAlignOf()) - head; + + ValidateSerializedLayout(&data, name); + + SerializeTraits<T>::Transfer (data, *this); + + if (copyData) + m_Context.pop(); +} + +template<class T> inline +void BlobWrite::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<class T> inline +void BlobWrite::TransferBasicData (T& srcData) +{ + Assert(m_Blob.size() >= GetActiveOffset() + sizeof(T)); + + // Write basic data into blob & endianswap + UInt8* blobPtr = GetActiveBlobPtr(); + memcpy(blobPtr, &srcData, sizeof(T)); + if (m_SwapEndianess) + SwapEndianBytes(*reinterpret_cast<T*> (blobPtr)); + + m_Context.top().m_Offset += sizeof(T); +} + +template<class T> inline +void BlobWrite::TransferPtr (bool isValidPtr, ReduceCopyData* reduceCopyData) +{ + TransferPtrImpl (isValidPtr, reduceCopyData, ALIGN_OF(T)); +} + +template<class T> inline +void BlobWrite::ReduceCopy (const ReduceCopyData& reduce) +{ + ReduceCopyImpl(reduce, ALIGN_OF(T)); +} + +#endif diff --git a/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp new file mode 100644 index 0000000..b66b941 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" +#include "BlobWriteTargetSupport.h" + +bool DoesBuildTargetSupportBlobification (BuildTargetPlatform target, TransferInstructionFlags flags) +{ + // If we are writing typetrees, then we can't use blobification + bool writeTypeTree = (flags & kDisableWriteTypeTree) == 0; + if (writeTypeTree) + return false; + + + // Webplayer & Editor should never use blobification + Assert(target != kBuildWebPlayerLZMA && target != kBuildWebPlayerLZMAStreamed && target != kBuildAnyPlayerData || target == kBuildNoTargetPlatform); + return true; +} + +bool IsBuildTarget64BitBlob (BuildTargetPlatform target) +{ + Assert(target != kBuildAnyPlayerData && target != kBuildWebPlayerLZMA && target != kBuildWebPlayerLZMAStreamed); + + // Building blob for the editor (Choose whatever we are running with) + if (target == kBuildNoTargetPlatform) + return sizeof(size_t) == sizeof(UInt64); + + // Known 64 bit platform? + bool target64Bit = target == kBuildMetroPlayerX64 || target == kBuildStandaloneWin64Player || target == kBuildStandaloneLinux64 || target == kBuildStandaloneLinuxUniversal; + if (target64Bit) + return true; + + return false; +}
\ No newline at end of file diff --git a/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h new file mode 100644 index 0000000..76191f5 --- /dev/null +++ b/Runtime/Serialize/Blobification/BlobWriteTargetSupport.h @@ -0,0 +1,6 @@ +#pragma once + +#include "Runtime/Serialize/SerializationMetaFlags.h" + +bool DoesBuildTargetSupportBlobification (BuildTargetPlatform target, TransferInstructionFlags flags); +bool IsBuildTarget64BitBlob (BuildTargetPlatform target);
\ No newline at end of file diff --git a/Runtime/Serialize/Blobification/OffsetPtrTest.cpp b/Runtime/Serialize/Blobification/OffsetPtrTest.cpp new file mode 100644 index 0000000..9ddd2a8 --- /dev/null +++ b/Runtime/Serialize/Blobification/OffsetPtrTest.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" + +#include "Runtime/Serialize/SerializeTraits.h" +#include "offsetptr.h" + + +#include <limits> + + +void TestOffsetPtr () +{ + OffsetPtr<size_t>* ptrHigh = new OffsetPtr<size_t>; + OffsetPtr<size_t>* ptrLow = new OffsetPtr<size_t>; + + size_t* ptrH = reinterpret_cast<size_t*>(std::numeric_limits<size_t>::max()-4); + size_t* ptrL = reinterpret_cast<size_t*>(4); + + ptrHigh->reset(ptrH); + ptrLow->reset(ptrL); + + size_t h = reinterpret_cast<size_t>(ptrHigh->Get()); + size_t l = reinterpret_cast<size_t>(ptrLow->Get()); + + + Assert(h == std::numeric_limits<size_t>::max()-4); + Assert(l == 4); + + delete ptrHigh; + delete ptrLow; + +} diff --git a/Runtime/Serialize/Blobification/ReduceCopyData.h b/Runtime/Serialize/Blobification/ReduceCopyData.h new file mode 100644 index 0000000..ae4952f --- /dev/null +++ b/Runtime/Serialize/Blobification/ReduceCopyData.h @@ -0,0 +1,8 @@ +#pragma once + +struct ReduceCopyData +{ + size_t ptrPosition; + size_t dataStart; + size_t blobSize; +}; diff --git a/Runtime/Serialize/Blobification/offsetptr.h b/Runtime/Serialize/Blobification/offsetptr.h new file mode 100644 index 0000000..7ef1809 --- /dev/null +++ b/Runtime/Serialize/Blobification/offsetptr.h @@ -0,0 +1,257 @@ +#pragma once + +#include "Runtime/mecanim/memory.h" +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/SerializeUtility.h" +#include "Runtime/Serialize/SerializeTraitsBase.h" +#include "Runtime/Serialize/TransferFunctionFwd.h" +#include "Runtime/Utilities/TypeUtilities.h" +#include "ReduceCopyData.h" + +template<typename TYPE> +class OffsetPtr +{ +public: + typedef TYPE value_type; + typedef TYPE* ptr_type; + typedef TYPE const* const_ptr_type; + typedef TYPE& reference_type; + typedef TYPE const& const_reference_type; + typedef size_t offset_type; + + OffsetPtr():m_Offset(0),m_DebugPtr(0) + { + } + + OffsetPtr (const OffsetPtr<value_type>& ptr):m_Offset(ptr.m_Offset) + { + } + + OffsetPtr& operator = (const OffsetPtr<value_type>& ptr) + { + m_Offset = ptr.m_Offset; + return *this; + } + + void reset(ptr_type ptr) + { + m_Offset = ptr != 0 ? reinterpret_cast<size_t>(ptr) - reinterpret_cast<size_t>(this) : 0; +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + } + + OffsetPtr& operator = (const ptr_type ptr) + { + reset (ptr); + return *this; + } + + ptr_type operator->() + { + ptr_type ptr = reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return ptr; + } + const_ptr_type operator->()const + { + return reinterpret_cast<const_ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); + } + + reference_type operator*() + { + ptr_type ptr = reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return *ptr; + } + + const_reference_type operator*()const + { + return *reinterpret_cast<const_ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); + } + + value_type& operator[](std::size_t i ) + { + assert(i != std::numeric_limits<std::size_t>::max()); + ptr_type ptr = reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); +#ifdef UNITY_EDITOR + m_DebugPtr = ptr; +#endif + return ptr[i]; + } + + value_type const& operator[](std::size_t i ) const + { + assert(i != std::numeric_limits<std::size_t>::max()); + const_ptr_type ptr = reinterpret_cast<const_ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); + return ptr[i]; + } + + bool IsNull()const + { + return m_Offset == 0; + } + + ptr_type Get() + { +#ifdef UNITY_EDITOR + m_DebugPtr = reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); +#endif + // TODO: serialize trait for offset ptr call begin and end which call OffsetPtr::Get + //Assert(!IsNull()); + return reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); + } + + const_ptr_type Get()const + { +#ifdef UNITY_EDITOR + m_DebugPtr = reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); +#endif + // TODO: serialize trait for offset ptr call begin and end which call OffsetPtr::Get + //Assert(!IsNull()); + return reinterpret_cast<ptr_type>(reinterpret_cast<std::size_t>(this) + m_Offset); + } + + + size_t get_size () const + { + return sizeof(TYPE); + } + +protected: + offset_type m_Offset; +#ifdef UNITY_EDITOR + mutable ptr_type m_DebugPtr; +#endif +}; + + +template<typename TYPE> +class SerializeTraits< OffsetPtr<TYPE> > : public SerializeTraitsBase< OffsetPtr<TYPE> > +{ + public: + + typedef OffsetPtr<TYPE> value_type; + inline static const char* GetTypeString (void*) { return "OffsetPtr"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return true; } + inline static bool AllowTransferOptimization () { return false; } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + if(IsSameType<TransferFunction, BlobWrite>::result) + { + ReduceCopyData reduce; + transfer.template TransferPtr<TYPE>(!data.IsNull(), &reduce); + if (!data.IsNull()) + { + transfer.Transfer(*data, "data"); + } + transfer.template ReduceCopy<TYPE> (reduce); + } + else if(transfer.IsReading () || transfer.IsWriting ()) + { + bool isNull = data.IsNull(); + + transfer.template TransferPtr<TYPE>(true, NULL); + if (isNull) + { + mecanim::memory::ChainedAllocator* allocator = static_cast<mecanim::memory::ChainedAllocator*> (transfer.GetUserData()); + data = allocator->Construct<TYPE>(); + } + + transfer.Transfer(*data, "data"); + } + else if(IsSameType<TransferFunction, BlobSize>::result) + { + transfer.template TransferPtr<TYPE>(false, NULL); + } + // Support for ProxyTransfer + else + { + transfer.template TransferPtr<TYPE>(false, NULL); + + TYPE p; + transfer.Transfer(p, "data"); + } + } +}; + +template<class T> +struct OffsetPtrArrayTransfer +{ + typedef T* iterator; + typedef T value_type; + + OffsetPtr<T>& m_Data; + UInt32& m_ArraySize; + void* m_Allocator; + bool m_ClearPtrs; + + OffsetPtrArrayTransfer (OffsetPtr<T>& data, UInt32& size, void* allocator, bool clearPtrs) + : m_Data(data),m_ArraySize(size) + { + m_Allocator = allocator; + m_ClearPtrs = clearPtrs; + } + + T* begin () { return m_Data.Get(); } + T* end () { return m_Data.Get() + m_ArraySize; } + size_t size() { return m_ArraySize; } + + void resize (int newSize) + { + m_ArraySize = newSize; + + mecanim::memory::ChainedAllocator* allocator = static_cast<mecanim::memory::ChainedAllocator*> (m_Allocator); + Assert(allocator != NULL); + + if (newSize != 0) + { + m_Data = allocator->ConstructArray<value_type> (newSize); + if (m_ClearPtrs) + memset(begin(), 0, sizeof(value_type) * newSize); + } + else + m_Data = NULL; + } +}; + +template<class T> +class SerializeTraits<OffsetPtrArrayTransfer<T> > : public SerializeTraitsBase<OffsetPtrArrayTransfer<T> > +{ +public: + + typedef OffsetPtrArrayTransfer<T> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + ReduceCopyData reduceCopy; + transfer.template TransferPtr<typename value_type::value_type>(transfer.IsReading() || transfer.IsWriting() ? data.m_ArraySize != 0 : false, &reduceCopy); + + transfer.TransferSTLStyleArray (data); + transfer.template ReduceCopy<typename value_type::value_type>(reduceCopy); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) + { + data.resize(rs); + } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + } +}; + + +#define MANUAL_ARRAY_TRANSFER2(TYPE,DATA,SIZE) OffsetPtrArrayTransfer<TYPE> DATA##ArrayTransfer (DATA, SIZE, transfer.GetUserData(), false); transfer.Transfer(DATA##ArrayTransfer, #DATA); +#define TRANSFER_BLOB_ONLY(DATA) if (IsSameType<TransferFunction, BlobWrite>::result || IsSameType<TransferFunction, BlobSize>::result) transfer.Transfer(DATA, #DATA); + diff --git a/Runtime/Serialize/BuildTargetVerification.h b/Runtime/Serialize/BuildTargetVerification.h new file mode 100644 index 0000000..5bfdb2f --- /dev/null +++ b/Runtime/Serialize/BuildTargetVerification.h @@ -0,0 +1,78 @@ +#ifndef BUILDTARGETVERIFICATION_H +#define BUILDTARGETVERIFICATION_H + +#include "SerializationMetaFlags.h" +#include "Configuration/UnityConfigure.h" + +inline bool IsPCStandaloneTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return + // We don't support building for these any more, but we still need the constants for asset bundle + // backwards compatibility. + targetPlatform == kBuildStandaloneOSXUniversal || + targetPlatform == kBuildStandaloneOSXPPC || + + targetPlatform == kBuildStandaloneOSXIntel || + targetPlatform == kBuildStandaloneOSXIntel64 || + targetPlatform == kBuildStandaloneWinPlayer || + targetPlatform == kBuildStandaloneWin64Player || + targetPlatform == kBuildMetroPlayerX86 || + targetPlatform == kBuildMetroPlayerX64 || + targetPlatform == kBuildMetroPlayerARM || + targetPlatform == kBuildStandaloneLinux || + targetPlatform == kBuildStandaloneLinux64 || + targetPlatform == kBuildStandaloneLinuxUniversal || + targetPlatform == kBuildWinGLESEmu; +} + +inline bool IsWebPlayerTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return targetPlatform == kBuildWebPlayerLZMA || targetPlatform == kBuildWebPlayerLZMAStreamed || targetPlatform == kBuildNaCl; +} + +inline bool IsMetroTargetPlatform (BuildTargetPlatform targetPlatform) +{ + return targetPlatform == kBuildMetroPlayerX86 || targetPlatform == kBuildMetroPlayerX64 || targetPlatform == kBuildMetroPlayerARM; +} + +inline bool CanLoadFileBuiltForTargetPlatform(BuildTargetPlatform targetPlatform) +{ + // Editor and BinaryToTextFile can load anything +#if UNITY_EDITOR || UNITY_EXTERNAL_TOOL + return true; + // Web players can handle all web formats +#elif WEBPLUG + return IsWebPlayerTargetPlatform(targetPlatform) || IsPCStandaloneTargetPlatform(targetPlatform); +#elif UNITY_METRO + // !! this code should be before #elif UNITY_WIN, because on Metro UNITY_WIN is defined as well !! + return IsMetroTargetPlatform(targetPlatform); +#elif UNITY_WP8 + // !! this code should be before #elif UNITY_WIN, because on WP8 UNITY_WIN is defined as well !! + return targetPlatform == kBuildWP8Player; +#elif UNITY_OSX || (UNITY_WIN && !UNITY_WINRT) || UNITY_LINUX + // Standalone can handle all web and standalone formats + return IsPCStandaloneTargetPlatform(targetPlatform) || IsWebPlayerTargetPlatform(targetPlatform); +#elif UNITY_WII + return targetPlatform == kBuildWii; +#elif UNITY_XENON + return targetPlatform == kBuildXBOX360; +#elif UNITY_PS3 + return targetPlatform == kBuildPS3; +#elif UNITY_IPHONE + return targetPlatform == kBuild_iPhone; +#elif UNITY_ANDROID + return targetPlatform == kBuild_Android; +#elif UNITY_FLASH + return targetPlatform == kBuildFlash; +#elif UNITY_WEBGL + return targetPlatform == kBuildWebGL; +#elif UNITY_BB10 + return targetPlatform == kBuildBB10; +#elif UNITY_TIZEN + return targetPlatform == kBuildTizen; +#else +#error Unknown platform +#endif +} + +#endif diff --git a/Runtime/Serialize/CacheWrap.cpp b/Runtime/Serialize/CacheWrap.cpp new file mode 100644 index 0000000..5ea4190 --- /dev/null +++ b/Runtime/Serialize/CacheWrap.cpp @@ -0,0 +1,461 @@ +#include "UnityPrefix.h" +#include "CacheWrap.h" +#include "FileCache.h" +#include "Runtime/Utilities/Utility.h" +#include <algorithm> +#if UNITY_EDITOR +#include "Editor/Src/GUIDPersistentManager.h" +#include "Runtime/Utilities/FileUtilities.h" +#endif + +#include "Runtime/Scripting/ScriptingUtility.h" + +using namespace std; + +#if UNITY_WINRT +#define OUTPUT_FIELDS_OF_LAST_SERIALIZABLE OutputFieldsOfLastSerializableObject(); +#else +#define OUTPUT_FIELDS_OF_LAST_SERIALIZABLE +#endif + +CachedReader::CachedReader () +{ + m_Cacher = 0; + m_Block = -1; + m_OutOfBoundsRead = false; + m_ActiveResourceImage = NULL; + #if CHECK_SERIALIZE_ALIGNMENT + m_CheckSerializeAlignment = true; + #endif +} + +void CachedReader::InitRead (CacheReaderBase& cacher, size_t position, size_t readSize) +{ + AssertIf (m_Block != -1); + m_Cacher = &cacher; + AssertIf (m_Cacher == NULL); + m_CacheSize = m_Cacher->GetCacheSize (); + m_Block = position / m_CacheSize; + m_MaximumPosition = position + readSize; + m_MinimumPosition = position; + + LockCacheBlockBounded (); + + SetPosition (position); +} + +void CachedReader::InitResourceImages (ResourceImageGroup& resourceImageGroup) +{ + Assert(m_ActiveResourceImage == NULL); + m_ResourceImageGroup = resourceImageGroup; +} + + +void CachedReader::LockCacheBlockBounded () +{ + m_Cacher->LockCacheBlock (m_Block, &m_CacheStart, &m_CacheEnd); + UInt8* maxPos = m_MaximumPosition - m_Block * m_CacheSize + m_CacheStart; + m_CacheEnd = min(m_CacheEnd, maxPos); +} + +size_t CachedReader::End () +{ + AssertIf (m_Block == -1); + size_t position = GetPosition (); + OutOfBoundsError (position, 0); + + m_Cacher->UnlockCacheBlock (m_Block); + m_Block = -1; + return position; +} + +void CachedReader::GetStreamingInfo (size_t offset, size_t size, StreamingInfo* streamingInfo) +{ + Assert (m_ActiveResourceImage != NULL || m_ActiveResourceImage == m_ResourceImageGroup.resourceImages[kStreamingResourceImage]); + + streamingInfo->offset = offset; + streamingInfo->size = size; + streamingInfo->path = m_ActiveResourceImage->GetStreamingPath(); +} + +CachedReader::~CachedReader () +{ + AssertIf (m_Block != -1); +} + +UInt8* CachedReader::FetchResourceImageData (size_t offset, size_t size) +{ + if (m_ActiveResourceImage == NULL) + { + ErrorString("Resource image for '" + m_Cacher->GetPathName() + "' couldn't be loaded!"); + return NULL; + } + + return m_ActiveResourceImage->Fetch (offset, size); +} + +ResourceImage::ResourceImage (const std::string& path, bool streaming) +{ + if (!streaming) + { + m_Size = GetFileLength(path); + m_Data = static_cast<UInt8*> (UNITY_MALLOC(kMemResource, m_Size)); + + if (!ReadFromFile(path, m_Data, 0, m_Size)) + { + ErrorString("Resource image couldn't be loaded completely"); + } + } + else + { + m_StreamingPath = path; + } +} + +ResourceImage::~ResourceImage () +{ + if (m_Data) + UNITY_FREE(kMemResource, m_Data); +} + +void CachedReader::Read (void* data, size_t size) +{ + if (m_CachePosition + size <= m_CacheEnd) + { + memcpy (data, m_CachePosition, size); + m_CachePosition += size; + } + else + { + // Read some data directly if it is coming in big chunks and we are not hitting the end of the file! + size_t position = GetPosition (); + OutOfBoundsError (position, size); + + if (m_OutOfBoundsRead) + { + memset(data, 0, size); + return; + } + + // Read enough bytes from the cache to align the position with the cache size + if (position % m_CacheSize != 0) + { + int blockEnd = ((position / m_CacheSize) + 1) * m_CacheSize; + int curReadSize = min<int> (size, blockEnd - position); + UpdateReadCache (data, curReadSize); + (UInt8*&)data += curReadSize; + size -= curReadSize; + position += curReadSize; + } + + // If we have a big block of data read directly without a cache, all aligned reads + int physicallyLimitedSize = min ((position + size), m_Cacher->GetFileLength ()) - position; + int blocksToRead = physicallyLimitedSize / m_CacheSize; + if (blocksToRead > 0) + { + int curReadSize = blocksToRead * m_CacheSize; + m_Cacher->DirectRead ((UInt8*)data, position, curReadSize); + m_CachePosition += curReadSize; + (UInt8*&)data += curReadSize; + size -= curReadSize; + } + + // Read the rest of the data from the cache! + while (size != 0) + { + int curReadSize = min<int> (size, m_CacheSize); + UpdateReadCache (data, curReadSize); + (UInt8*&)data += curReadSize; + size -= curReadSize; + } + } +} + +void CachedReader::Skip (int size) +{ + if (m_CachePosition + size <= m_CacheEnd) + { + m_CachePosition += size; + } + else + { + int position = GetPosition (); + SetPosition(position + size); + } +} + +void memcpy_constrained_src (void* dst, const void* src, int size, const void* srcFrom, void* srcTo); +void memcpy_constrained_dst (void* dst, const void* src, int size, const void* dstFrom, void* dstTo); + +void memcpy_constrained_src (void* dst, const void* src, int size, const void* srcFrom, void* srcTo) +{ + UInt8* fromClamped = clamp ((UInt8*)src, (UInt8*)srcFrom, (UInt8*)srcTo); + UInt8* toClamped = clamp ((UInt8*)src + size, (UInt8*)srcFrom, (UInt8*)srcTo); + + int offset = fromClamped - (UInt8*)src; + size = toClamped - fromClamped; + memcpy ((UInt8*)dst + offset, (UInt8*)src + offset, size); +} + +void memcpy_constrained_dst (void* dst, const void* src, int size, const void* dstFrom, void* dstTo) +{ + UInt8* fromClamped = clamp ((UInt8*)dst, (UInt8*)dstFrom, (UInt8*)dstTo); + UInt8* toClamped = clamp ((UInt8*)dst + size, (UInt8*)dstFrom, (UInt8*)dstTo); + + int offset = fromClamped - (UInt8*)dst; + size = toClamped - fromClamped; + memcpy ((UInt8*)dst + offset, (UInt8*)src + offset, size); +} + +void CachedReader::UpdateReadCache (void* data, size_t size) +{ + AssertIf (m_Cacher == NULL); + AssertIf (size > m_CacheSize); + + size_t position = GetPosition (); + OutOfBoundsError(position, size); + + if (m_OutOfBoundsRead) + { + memset(data, 0, size); + return; + } + + // copy data oldblock + SetPosition (position); + memcpy_constrained_src (data, m_CachePosition, size, m_CacheStart, m_CacheEnd); + + // Read next cache block only if we actually need it. + if (m_CachePosition + size > m_CacheEnd) + { + // Check if the cache block + // copy data new block + SetPosition (position + size); + UInt8* cachePosition = position - m_Block * m_CacheSize + m_CacheStart; + memcpy_constrained_src (data, cachePosition, size, m_CacheStart, m_CacheEnd); + } + else + { + m_CachePosition += size; + } +} + +std::string GetNicePath (const CacheReaderBase& cacher) +{ + #if UNITY_EDITOR + string path = cacher.GetPathName(); + string assetPath = AssetPathNameFromAnySerializedPath(GetProjectRelativePath(cacher.GetPathName())); + + if (!assetPath.empty()) + return cacher.GetPathName() + "\' - \'" + assetPath; + else + return cacher.GetPathName(); + #else + return cacher.GetPathName(); + #endif +} + +void CachedReader::OutOfBoundsError (size_t position, size_t size) +{ + if (m_OutOfBoundsRead) + return; + + #define ERROR_FMT "The file \'%s\' is corrupted! Remove it and launch unity again!\n" \ + "[Position out of bounds! %" PRINTF_SIZET_FORMAT " %s %" PRINTF_SIZET_FORMAT "]" + + if (position + size > m_Cacher->GetFileLength ()) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, ">", m_Cacher->GetFileLength ()); + m_OutOfBoundsRead = true; + } + + if (position + size > m_MaximumPosition) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, ">", m_MaximumPosition); + m_OutOfBoundsRead = true; + } + + if (position < m_MinimumPosition) + { + OUTPUT_FIELDS_OF_LAST_SERIALIZABLE; + FatalErrorMsg (ERROR_FMT, GetNicePath(*m_Cacher).c_str(), position + size, "<", m_MinimumPosition); + m_OutOfBoundsRead = true; + } +} + +void CachedReader::SetPosition (size_t position) +{ + OutOfBoundsError(position, 0); + if (m_OutOfBoundsRead) + return; + + if (position / m_CacheSize != m_Block) + { + m_Cacher->UnlockCacheBlock (m_Block); + m_Block = position / m_CacheSize; + m_Cacher->LockCacheBlock (m_Block, &m_CacheStart, &m_CacheEnd); + } + m_CachePosition = position - m_Block * m_CacheSize + m_CacheStart; +} + +void CachedReader::Align4Read () +{ + UInt32 offset = m_CachePosition - m_CacheStart; + offset = ((offset + 3) >> 2) << 2; + m_CachePosition = m_CacheStart + offset; +} + + +////// + +void CachedWriter::InitActiveWriter (ActiveWriter& activeWriter, CacheWriterBase& cacher) +{ + Assert (activeWriter.block == -1); + Assert (&cacher != NULL); + + activeWriter.cacheBase = &cacher; + activeWriter.block = 0; + activeWriter.cacheBase->LockCacheBlock (activeWriter.block, &activeWriter.cacheStart, &activeWriter.cacheEnd); + activeWriter.cachePosition = activeWriter.cacheStart; +} + +void CachedWriter::Align4Write () +{ + UInt32 leftOver = Align4LeftOver (m_ActiveWriter.cachePosition - m_ActiveWriter.cacheStart); + UInt8 value = 0; + for (UInt32 i=0;i<leftOver;i++) + Write(value); +} + +void CachedWriter::Write (const void* data, size_t size) +{ + if (m_ActiveWriter.cachePosition + size < m_ActiveWriter.cacheEnd) + { + memcpy (m_ActiveWriter.cachePosition, data, size); + m_ActiveWriter.cachePosition += size; + } + else + { + while (size != 0) + { + size_t curWriteSize = min (size, m_ActiveWriter.cacheBase->GetCacheSize()); + UpdateWriteCache (data, curWriteSize); + (UInt8*&)data += curWriteSize; + size -= curWriteSize; + } + } +} + +void CachedWriter::UpdateWriteCache (const void* data, size_t size) +{ + Assert (m_ActiveWriter.cacheBase != NULL); + AssertIf (size > m_ActiveWriter.cacheBase->GetCacheSize()); + + size_t position = GetPosition (); + size_t cacheSize = m_ActiveWriter.cacheBase->GetCacheSize(); + // copy data from oldblock + memcpy_constrained_dst (m_ActiveWriter.cachePosition, data, size, m_ActiveWriter.cacheStart, m_ActiveWriter.cacheEnd); + + SetPosition (position + size); + + // copy data new block + UInt8* cachePosition = position - m_ActiveWriter.block * cacheSize + m_ActiveWriter.cacheStart; + memcpy_constrained_dst (cachePosition, data, size, m_ActiveWriter.cacheStart, m_ActiveWriter.cacheEnd); +} + +void CachedWriter::SetPosition (size_t position) +{ + size_t cacheSize = m_ActiveWriter.cacheBase->GetCacheSize(); + int newBlock = position / cacheSize; + if (newBlock != m_ActiveWriter.block) + { + Assert(newBlock == m_ActiveWriter.block + 1); + + m_ActiveWriter.cacheBase->UnlockCacheBlock (m_ActiveWriter.block); + m_ActiveWriter.block = newBlock; + m_ActiveWriter.cacheBase->LockCacheBlock (m_ActiveWriter.block, &m_ActiveWriter.cacheStart, &m_ActiveWriter.cacheEnd); + } + m_ActiveWriter.cachePosition = position - m_ActiveWriter.block * cacheSize + m_ActiveWriter.cacheStart; +} + +size_t CachedWriter::GetPosition () const +{ + return m_ActiveWriter.GetPosition(); +} + + +bool CachedWriter::CompleteWriting () +{ + m_ActiveWriter.cacheBase->UnlockCacheBlock (m_ActiveWriter.block); + + bool success = m_ActiveWriter.cacheBase->CompleteWriting (m_ActiveWriter.GetPosition()); + + #if UNITY_EDITOR + if (m_ActiveResourceImageMode != kResourceImageNotSupported) + { + for (int i=0;i<kNbResourceImages;i++) + { + success &= m_ResourceImageWriters[i].cacheBase->CompleteWriting (m_ResourceImageWriters[i].GetPosition()); + success &= m_ResourceImageWriters[i].cacheBase->WriteHeaderAndCloseFile (NULL, 0, 0); + } + } + #endif + + return success; +} + +size_t CachedWriter::ActiveWriter::GetPosition () const +{ + return cachePosition - cacheStart + block * cacheBase->GetCacheSize(); +} + +#if UNITY_EDITOR + +void CachedWriter::InitWrite (CacheWriterBase& cacher) +{ + InitActiveWriter(m_ActiveWriter, cacher); + m_DefaultWriter = m_ActiveWriter; + m_ActiveResourceImageMode = kResourceImageNotSupported; +} + +void CachedWriter::InitResourceImage (ActiveResourceImage index, CacheWriterBase& resourceImage) +{ + m_ActiveResourceImageMode = kResourceImageInactive; + InitActiveWriter(m_ResourceImageWriters[index], resourceImage); +} + +void CachedWriter::BeginResourceImage (ActiveResourceImage resourceImage) +{ + if (m_ActiveResourceImageMode == kResourceImageNotSupported) + return; + + Assert(m_ActiveResourceImageMode == kResourceImageInactive); + Assert(resourceImage > kResourceImageInactive); + + m_ActiveResourceImageMode = resourceImage; + + m_DefaultWriter = m_ActiveWriter; + m_ActiveWriter = m_ResourceImageWriters[m_ActiveResourceImageMode]; + Assert(m_ActiveWriter.cacheBase != NULL); +} + +void CachedWriter::EndResourceImage () +{ + Assert(IsWritingResourceImage()); + + m_ResourceImageWriters[m_ActiveResourceImageMode] = m_ActiveWriter; + m_ActiveWriter = m_DefaultWriter; + + m_ActiveResourceImageMode = kResourceImageInactive; +} + +#else + +void CachedWriter::InitWrite (CacheWriterBase& cacher) +{ + InitActiveWriter(m_ActiveWriter, cacher); +} + +#endif diff --git a/Runtime/Serialize/CacheWrap.h b/Runtime/Serialize/CacheWrap.h new file mode 100644 index 0000000..6ded041 --- /dev/null +++ b/Runtime/Serialize/CacheWrap.h @@ -0,0 +1,229 @@ +#ifndef CACHEWRAP_H +#define CACHEWRAP_H + +#include "Configuration/UnityConfigure.h" +#include "Runtime/Serialize/FileCache.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + +class CachedWriter +{ + struct ActiveWriter + { + UInt8* cachePosition; + UInt8* cacheStart; + UInt8* cacheEnd; + SInt32 block; + CacheWriterBase* cacheBase; + + ActiveWriter () { cachePosition = NULL; cacheStart = NULL; cacheEnd = NULL; block = -1; cacheBase = NULL; } + size_t GetPosition () const; + }; + + ActiveWriter m_ActiveWriter; + + #if UNITY_EDITOR + + ActiveResourceImage m_ActiveResourceImageMode; + + ActiveWriter m_DefaultWriter; + ActiveWriter m_ResourceImageWriters[kNbResourceImages]; + + #endif + + static void InitActiveWriter (CachedWriter::ActiveWriter& activeWriter, CacheWriterBase& cacher); + void SetPosition (size_t position); + void EXPORT_COREMODULE UpdateWriteCache (const void* data, size_t size); + + public: + + void InitWrite (CacheWriterBase& cacher); + + bool CompleteWriting (); + + +#if UNITY_EDITOR + void InitResourceImage (ActiveResourceImage index, CacheWriterBase& resourceImage); + + void BeginResourceImage (ActiveResourceImage resourceImageType); + void EndResourceImage (); + bool IsWritingResourceImage () { return m_ActiveResourceImageMode > kResourceImageInactive; } + + CacheWriterBase& GetCacheBase () { return *m_ActiveWriter.cacheBase; } + +#endif + + template<class T> + void Write (const T& data) + { +#if CHECK_SERIALIZE_ALIGNMENT + if (m_CheckSerializeAlignment) + { + SInt32 position = reinterpret_cast<SInt32>(m_ActiveWriter.cachePosition); + SInt32 size = sizeof(T); + SInt32 align = position % size; + if (align != 0) + { + ErrorString("Alignment error "); + } + } +#endif + + if (m_ActiveWriter.cachePosition + sizeof (T) < m_ActiveWriter.cacheEnd) + { + *reinterpret_cast<T*> (m_ActiveWriter.cachePosition) = data; + m_ActiveWriter.cachePosition += sizeof (T); + } + else + UpdateWriteCache (&data, sizeof (data)); + } + + void Align4Write (); + + void Write (const void* data, size_t size); + + size_t GetPosition () const; +}; + +struct StreamingInfo +{ + size_t offset; + size_t size; + std::string path; + + bool IsValid () const { return !path.empty(); } + + StreamingInfo () { offset = 0; size = 0; } +}; + + +struct ResourceImage +{ + UInt8* m_Data; + UInt32 m_Size; + std::string m_StreamingPath; + + public: + + ResourceImage (const std::string& path, bool stream); + ~ResourceImage (); + + UInt8* Fetch (size_t offset, size_t size) + { + Assert(m_Data != NULL); + Assert(size + offset <= m_Size); + return m_Data + offset; + } + + const std::string& GetStreamingPath () { Assert(!m_StreamingPath.empty()); return m_StreamingPath; } +}; + +struct EXPORT_COREMODULE ResourceImageGroup +{ + ResourceImage* resourceImages[kNbResourceImages]; + + ResourceImageGroup () { memset(this, 0, sizeof(ResourceImageGroup)); } +}; + +class EXPORT_COREMODULE CachedReader +{ + private: + + UInt8* m_CachePosition; + UInt8* m_CacheStart; + UInt8* m_CacheEnd; + CacheReaderBase* m_Cacher; + SInt32 m_Block; + size_t m_CacheSize; + size_t m_MinimumPosition; + size_t m_MaximumPosition; + bool m_OutOfBoundsRead; + + ResourceImage* m_ActiveResourceImage; + ResourceImageGroup m_ResourceImageGroup; + + void UpdateReadCache (void* data, size_t size); + + CachedReader (const CachedReader& c);// undefined + CachedReader& operator = (const CachedReader& c);// undefined + + void OutOfBoundsError (size_t position, size_t size); + void LockCacheBlockBounded(); + + public: + + CachedReader (); + ~CachedReader (); + + void InitRead (CacheReaderBase& cacher, size_t position, size_t size); + void InitResourceImages (ResourceImageGroup& resourceImage); + + size_t GetEndPosition () { return m_MaximumPosition; } + + size_t End (); + + template<class T> + void Skip () + { + m_CachePosition += sizeof (T); + } + + void Skip (int size); + + UInt8* FetchResourceImageData (size_t offset, size_t size); + + void GetStreamingInfo (size_t offset, size_t size, StreamingInfo* streamingInfo); + + void BeginResourceImage (ActiveResourceImage index) { m_ActiveResourceImage = m_ResourceImageGroup.resourceImages[index]; } + void EndResourceImage () { m_ActiveResourceImage = NULL; } + bool IsReadingResourceImage () { return m_ActiveResourceImage != NULL; } + const char* GetSerializedFilePathName() { return m_Cacher->GetPathName().c_str(); } + + template<class T> + void Read (T& data, size_t position) + { + m_CachePosition = m_CacheStart + position - m_Block * m_CacheSize; + if (m_CachePosition >= m_CacheStart && m_CachePosition + sizeof (data) <= m_CacheEnd) + { + data = *reinterpret_cast<T*> (m_CachePosition); + m_CachePosition += sizeof (T); + } + else + UpdateReadCache (&data, sizeof (data)); + } + + template<class T> + void Read (T& data) + { + if (m_CachePosition + sizeof (T) <= m_CacheEnd) + { + data = *reinterpret_cast<T*> (m_CachePosition); + m_CachePosition += sizeof (T); + } + else + UpdateReadCache (&data, sizeof (data)); + } + + void Align4Read (); + + size_t GetPosition () const { return m_CachePosition - m_CacheStart + m_Block * m_CacheSize; } + void SetPosition (size_t position); + void SetAbsoluteMemoryPosition(UInt8* position) { m_CachePosition = position; } + UInt8* GetAbsoluteMemoryPosition() { return m_CachePosition; } + + void Read (void* data, size_t size); + + CacheReaderBase* GetCacher () const { return m_Cacher; } +}; + +inline UInt32 Align4 (UInt32 size) +{ + UInt32 value = ((size + 3) >> 2) << 2; + return value; +} + +inline UInt32 Align4LeftOver (UInt32 size) +{ + return Align4(size) - size; +} + +#endif diff --git a/Runtime/Serialize/DumpSerializedDataToText.cpp b/Runtime/Serialize/DumpSerializedDataToText.cpp new file mode 100644 index 0000000..1be6ddf --- /dev/null +++ b/Runtime/Serialize/DumpSerializedDataToText.cpp @@ -0,0 +1,372 @@ +#include "UnityPrefix.h" +#include "DumpSerializedDataToText.h" +#include "TypeTree.h" +#include "CacheWrap.h" +#include "Runtime/Utilities/GUID.h" + +using namespace std; + +#if (UNITY_EDITOR || UNITY_INCLUDE_SERIALIZATION_DUMP || DEBUGMODE) + +void DumpSerializedDataToText (const TypeTree& typeTree, dynamic_array<UInt8>& data) +{ + int offset = 0; + RecursiveOutput(typeTree, data.begin(), &offset, 0, cout, kDumpNormal, 0, false, -1); +} +#define TAB for (int t=0;t<tab;t++) os << '\t'; + +template<class T> +void DoSwap (T& t, bool swapBytes) +{ + if (swapBytes) + SwapEndianBytes(t); +} + + +SInt32 CalculateByteSize(const TypeTree& type) { + if(type.m_ByteSize != -1) + return type.m_ByteSize; + + SInt32 r=0; + for (TypeTree::const_iterator i=type.begin ();i != type.end ();i++) + r+=CalculateByteSize(*i); + return r; +} + +void OutputType (const TypeTree& type, ostream& os) +{ + os << "Name: " << type.m_Name; + os << " Type: " << type.m_Type; + os << " ByteSize: " << type.m_ByteSize; + os << " TypeTreePosition: " << type.m_Index; + os << " IsArray: " << type.m_IsArray; + os << " Version: " << type.m_Version; + os << " MetaFlag: " << type.m_MetaFlag; + os << " IsArray: " << type.m_IsArray; +} + +string ExtractString (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + string value; + SInt32 size = *reinterpret_cast<const SInt32*> (data + *offset); + DoSwap(size, doSwap); + value.reserve (size); + for (int i=0;i<size;i++) + { + value += data[*offset + i + sizeof(SInt32)]; + } + + *offset += sizeof(SInt32) + size; + + if (type.m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) + *offset = Align4(*offset); + + return value; +} + +string ExtractMdFour (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + string value; + SInt32 size = *reinterpret_cast<const SInt32*> (data + *offset); + DoSwap(size, doSwap); + value.reserve (size*2); + for (int i=0;i<size;i++) + { + value += Format("%02x", data[*offset + i + sizeof(SInt32)]); + } + + *offset += sizeof(SInt32) + size; + + if (type.m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) + *offset = Align4(*offset); + + return value; +} + +string ExtractVector (const TypeTree& type, const UInt8* data, int* offset, bool doSwap, int dimension) +{ + AssertIf(type.m_Father == NULL); + AssertIf(type.m_Children.size() != dimension); + AssertIf(CalculateByteSize(type) != dimension*4); + + string val = "("; + for (int i = 0; i < dimension; ++i) + { + float v = *reinterpret_cast<const float*> (data + *offset); + DoSwap(v, doSwap); + if (i != 0) + val += ' '; + val += Format("%g", v); + *offset += 4; + } + val += ')'; + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return val; +} + +string ExtractRectOffset (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + AssertIf(type.m_Father == NULL); + AssertIf(type.m_Children.size() != 4); + + string val = "("; + TypeTree::TypeTreeList::const_iterator it = type.m_Children.begin(); + for (int i = 0; i < 4; ++i, ++it) + { + int v = *reinterpret_cast<const int*> (data + *offset); + DoSwap(v, doSwap); + if (i != 0) + val += ' '; + val += Format("%s %i", it->m_Name.c_str(), v); + *offset += 4; + } + val += ')'; + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return val; +} + + +string ExtractPPtr (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + SInt32 fileID = *reinterpret_cast<const SInt32*>(data + *offset); + SInt32 pathID = *reinterpret_cast<const SInt32*>(data + *offset + 4); + DoSwap(fileID, doSwap); + DoSwap(pathID, doSwap); + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + *offset += 8; + + return Format ("(file %i path %i)", (int)fileID, (int)pathID); +} + +string ExtractGUID (const TypeTree& type, const UInt8* data, int* offset, bool doSwap) +{ + AssertIf(type.m_Father == NULL); + AssertIf(type.m_Children.size() != 4); + AssertIf(CalculateByteSize(type) != 4*4); + + UnityGUID val; + for (int i = 0; i < 4; ++i) { + UInt32 v = *reinterpret_cast<const UInt32*> (data + *offset); + val.data[i]=v; + *offset += 4; + } + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + + return GUIDToString(val); +} + + +void OutputValue (const TypeTree& type, const UInt8* data, int* offset, ostream& os, bool doSwap) +{ +#define OUTPUT(x) \ +else if (type.m_Type == #x) \ +{ \ +x value = *reinterpret_cast<const x*> (data + *offset); \ +DoSwap(value, doSwap);\ +os << value; \ +} + +#define OUTPUT_INT(x) \ +else if (type.m_Type == #x) \ +{ \ +x value = *reinterpret_cast<const x*> (data + *offset); \ +DoSwap(value, doSwap);\ +int intValue = value; \ +os << intValue; \ +} + + + if (false) { } + OUTPUT (float) + OUTPUT (double) + OUTPUT (int) + OUTPUT (unsigned int) + OUTPUT (SInt32) + OUTPUT (UInt32) + OUTPUT (SInt16) + OUTPUT (UInt16) + OUTPUT (SInt64) + OUTPUT (UInt64) + OUTPUT_INT (SInt8) + OUTPUT_INT (UInt8) + OUTPUT (char) + OUTPUT (bool) + else + { + AssertString ("Unsupported type! " + type.m_Type); + } + *offset = *offset + type.m_ByteSize; +} + +const int kArrayMemberColumns = 25; + +void RecursiveOutput (const TypeTree& type, const UInt8* data, int* offset, int tab, ostream& os, DumpOutputMode mode, int pathID, bool doSwap, int arrayIndex) +{ + if (type.m_Type == "Vector3f" && type.m_ByteSize != 12) + { + AssertString ("Unsupported type! " + type.m_Type); + } + + if (!type.m_IsArray && arrayIndex == -1 && mode != kDumpClean) + os << Format("% 5d: ", (int)type.m_Index); + + if (type.IsBasicDataType ()) + { + // basic data type + if (arrayIndex == -1) + { + TAB os << type.m_Name << " "; + OutputValue (type, data, offset, os, doSwap); + os << " (" << type.m_Type << ")"; + os << endl; + } + else + { + // array members: multiple members per line + if (arrayIndex % kArrayMemberColumns == 0) + { + if (arrayIndex != 0) + os << endl; + TAB os << type.m_Name << " (" << type.m_Type << ") #" << arrayIndex << ": "; + OutputValue (type, data, offset, os, doSwap); + } + else + { + os << ' '; + OutputValue (type, data, offset, os, doSwap); + } + } + } + else if (type.m_IsArray) + { + // Extract and Print size + int size = *reinterpret_cast<const SInt32*> (data + *offset); + DoSwap(size, doSwap); + + RecursiveOutput (type.m_Children.front (), data, offset, tab, os, mode, 0, doSwap, -1); + // Print children + for (int i=0;i<size;i++) + { + // char buffy[64]; sprintf (buffy, "%s[%d]", type.m_Name.c_str (), i); + RecursiveOutput (type.m_Children.back (), data, offset, tab, os, mode, 0, doSwap, i); + } + os << endl; + } + else if (type.m_Type == "string") + { + TAB os << type.m_Name << " "; + os << "\""<< ExtractString (type, data, offset, doSwap) << "\""; + + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "Vector4f" && type.m_ByteSize == 16) + { + TAB os << type.m_Name << " "; + os << ExtractVector (type, data, offset, doSwap, 4); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "Vector3f" && type.m_ByteSize == 12) + { + TAB os << type.m_Name << " "; + os << ExtractVector (type, data, offset, doSwap, 3); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "Vector2f" && type.m_ByteSize == 8) + { + TAB os << type.m_Name << " "; + os << ExtractVector (type, data, offset, doSwap, 2); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "ColorRGBA" && type.m_ByteSize == 16) + { + TAB os << type.m_Name << " "; + os << ExtractVector (type, data, offset, doSwap, 4); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "FastPropertyName" && type.m_Children.size()==1 && type.m_Children.front().m_Type=="string") + { + TAB os << type.m_Name << " "; + os << "\""<< ExtractString (type, data, offset, doSwap) << "\""; + os << " (" << type.m_Type << ")" << endl; + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } + } + else if (type.m_Type == "RectOffset" && type.m_ByteSize == 16) + { + TAB os << type.m_Name << " "; + os << ExtractRectOffset (type, data, offset, doSwap); + os << " (" << type.m_Type << ")" << endl; + } + else if (BeginsWith(type.m_Type, "PPtr<") && type.m_ByteSize == 8) + { + TAB os << type.m_Name << " "; + os << ExtractPPtr (type, data, offset, doSwap); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "GUID") + { + TAB os << type.m_Name << " "; + os << ExtractGUID (type, data, offset, doSwap); + os << " (" << type.m_Type << ")" << endl; + } + else if (type.m_Type == "MdFour") + { + TAB os << type.m_Name << " "; + os << ExtractMdFour (type, data, offset, doSwap); + os << " (" << type.m_Type << ")" << endl; + } + else + { + TAB + if (type.m_Father != NULL) + { + os << type.m_Name << " "; + os << " (" << type.m_Type << ")" ; + } + else + { + os << type.m_Type ; + } + if (mode != kDumpClean) + { + if (pathID == 0) + os << Format(" [size: %d, children: %d]", (int)CalculateByteSize(type), (int)type.m_Children.size()); + else + os << Format(" [size: %d, children: %d pathID: %d]", (int)CalculateByteSize(type), (int)type.m_Children.size(), pathID); + } + os << endl; + + tab++; + for (TypeTree::const_iterator i=type.begin ();i != type.end ();i++) + RecursiveOutput (*i, data, offset, tab, os, mode, 0, doSwap, -1); + tab--; + } + + if (type.m_MetaFlag & kAlignBytesFlag) + { + *offset = Align4(*offset); + } +} + +#endif // UNITY_EDITOR || UNITY_INCLUDE_SERIALIZATION_DUMP diff --git a/Runtime/Serialize/DumpSerializedDataToText.h b/Runtime/Serialize/DumpSerializedDataToText.h new file mode 100644 index 0000000..f78cb81 --- /dev/null +++ b/Runtime/Serialize/DumpSerializedDataToText.h @@ -0,0 +1,24 @@ +#pragma once + +#ifndef UNITY_INCLUDE_SERIALIZATION_DUMP +#define UNITY_INCLUDE_SERIALIZATION_DUMP 0 +#endif + +// Debugging functions that dump the state of an object or typetree/bytearray to stdout +// Used in BinaryToTextFile by defining UNITY_INCLUDE_SERIALIZATION_DUMP +#if UNITY_EDITOR || UNITY_INCLUDE_SERIALIZATION_DUMP || DEBUGMODE +#include <iostream> +#include "Runtime/Utilities/dynamic_array.h" + +class TypeTree; + +enum DumpOutputMode +{ + kDumpNormal, + kDumpClean, +}; + +void DumpSerializedDataToText (const TypeTree& typeTree, dynamic_array<UInt8>& data); +void RecursiveOutput (const TypeTree& type, const UInt8* data, int* offset, int tab, std::ostream& os, DumpOutputMode mode, int pathID, bool doSwap, int arrayIndex); +#endif + diff --git a/Runtime/Serialize/FileCache.cpp b/Runtime/Serialize/FileCache.cpp new file mode 100644 index 0000000..c297ed6 --- /dev/null +++ b/Runtime/Serialize/FileCache.cpp @@ -0,0 +1,461 @@ +#include "UnityPrefix.h" +#include "FileCache.h" +#include "Runtime/Utilities/algorithm_utility.h" +#include "Runtime/Utilities/Utility.h" +#include "Runtime/Utilities/PathNameUtility.h" + +#if SUPPORT_SERIALIZE_WRITE +#include "Runtime/Utilities/FileUtilities.h" +#endif + +using namespace std; + +#define USE_OPEN_FILE_CACHE UNITY_EDITOR + +#if USE_OPEN_FILE_CACHE + +struct OpenFilesCache +{ + enum { kOpenedFileCacheCount = 5 }; + File* m_Cache[kOpenedFileCacheCount]; + UInt32 m_TimeStamps[kOpenedFileCacheCount]; + UInt32 m_TimeStamp; + + OpenFilesCache () + { + m_TimeStamp = 0; + for (int i=0;i<kOpenedFileCacheCount;i++) + { + m_Cache[i] = NULL; + m_TimeStamps[i] = 0; + } + } + + void OpenCached (File* theFile, const std::string& thePath) + { + m_TimeStamp++; + + // find cache, don't do anything if we are in the cache + for (int i=0;i<kOpenedFileCacheCount;i++) + { + if (theFile == m_Cache[i]) + { + m_TimeStamps[i] = m_TimeStamp; + return; + } + } + + // Find Least recently used cache entry + UInt32 lruTimeStamp = m_TimeStamps[0]; + int lruIndex = 0; + for (int i=1;i<kOpenedFileCacheCount;i++) + { + if (m_TimeStamps[i] < lruTimeStamp) + { + lruTimeStamp = m_TimeStamps[i]; + lruIndex = i; + } + } + + // replace the least recently used cache entry + if (m_Cache[lruIndex] != NULL) + { + #if UNITY_OSX + m_Cache[lruIndex]->Lock (File::kNone, false); + #endif + m_Cache[lruIndex]->Close (); + } + + m_Cache[lruIndex] = theFile; + m_TimeStamps[lruIndex] = m_TimeStamp; + + if(!theFile->Open (thePath, File::kReadPermission, File::kSilentReturnOnOpenFail)) + ErrorString(Format("Could not open file %s for read", thePath.c_str())); + + #if UNITY_OSX + theFile->Lock (File::kShared, false); + #endif + } + + void ForceCloseAll () + { + // Find and close cache + for (int i=0;i<kOpenedFileCacheCount;i++) + { + if (m_Cache[i] != NULL) + ForceClose (m_Cache[i]); + } + } + + void ForceClose (File* cachable) + { + // Find and close cache + for (int i=0;i<kOpenedFileCacheCount;i++) + { + if (m_Cache[i] == cachable) + { + #if UNITY_OSX + m_Cache[i]->Lock (File::kNone, false); + #endif + m_Cache[i]->Close (); + m_Cache[i] = NULL; + m_TimeStamps[i] = 0; + return; + } + } + } +}; +OpenFilesCache gOpenFilesCache; + +void ForceCloseAllOpenFileCaches () +{ + gOpenFilesCache.ForceCloseAll (); +} + +#endif + + +CacheReaderBase::~CacheReaderBase () +{} + +CacheWriterBase::~CacheWriterBase () +{} + +FileCacherRead::FileCacherRead (const string& pathName, size_t cacheSize, size_t cacheCount) +{ + m_RootHeader = GET_CURRENT_ALLOC_ROOT_HEADER(); + m_Path = PathToAbsolutePath(pathName); + + // Initialize Cache + m_MaxCacheCount = cacheCount; + m_CacheSize = cacheSize; + m_TimeStamp = 0; + + // Get File size + m_FileSize = ::GetFileLength(m_Path); + + #if !USE_OPEN_FILE_CACHE + if(!m_File.Open(m_Path, File::kReadPermission, File::kSilentReturnOnOpenFail)) + ErrorString(Format("Could not open file %s for read", m_Path.c_str())); + + // Make the file non-overwritable by the cache (to make caching behaviour imitate windows) + #if UNITY_OSX + m_File.Lock (File::kShared, false); + #endif + + #endif + #if DEBUG_LINEAR_FILE_ACCESS + m_LastFileAccessPosition = 0; + #endif +} + +FileCacherRead::~FileCacherRead () +{ + for (CacheBlocks::iterator i = m_CacheBlocks.begin ();i != m_CacheBlocks.end ();++i) + { + AssertIf (i->second.lockCount); + UNITY_FREE(kMemFile,i->second.data); + } + + m_CacheBlocks.clear (); + + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.ForceClose (&m_File); + #else + // Make the file overwritable by the cache. + #if UNITY_OSX + m_File.Lock (File::kNone, false); + #endif + + m_File.Close(); + + #endif +} + +void FileCacherRead::ReadCacheBlock (CacheBlock& cacheBlock) +{ + int block = cacheBlock.block; + // Watch out for not reading over eof + int readSize = min<int> (m_CacheSize, m_FileSize - block * m_CacheSize); + + // load the data from disk + // only if the physical file contains any data for this block + if (readSize > 0) + { + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.OpenCached (&m_File, m_Path); + #endif + + #if DEBUG_LINEAR_FILE_ACCESS + size_t position = block * m_CacheSize; + #endif + + m_File.Read (block * m_CacheSize, cacheBlock.data, readSize); + + #if DEBUG_LINEAR_FILE_ACCESS + // printf_console("ACCESS: [%08x] %s %i bytes @ %i to %08x\n",this, __FUNCTION__, readSize, block * m_CacheSize, cacheBlock.data); + std::string fileName = GetLastPathNameComponent(m_Path); + if (position < m_LastFileAccessPosition && fileName != "mainData" && fileName != "unity default resources") + { + ErrorString(Format("File access: %s is not linear Reading: %d Seek position: %d", fileName.c_str(), position, m_LastFileAccessPosition)); + } + + m_LastFileAccessPosition = position + readSize; + #endif + } +} + +void FileCacherRead::DirectRead (void* data, size_t position, size_t size) +{ + // load the data from disk + // only if the physical file contains any data for this block + FatalErrorIf (m_FileSize - position < size); + + #if USE_OPEN_FILE_CACHE + gOpenFilesCache.OpenCached (&m_File, m_Path); + #endif + + m_File.Read (position, data, size); + + #if DEBUG_LINEAR_FILE_ACCESS + // printf_console("ACCCESS: [%08x] %s %i bytes @ %i to %08x\n",this, __FUNCTION__, size, position, data); + std::string fileName = GetLastPathNameComponent(m_Path); + if (position < m_LastFileAccessPosition && fileName != "mainData" && fileName != "unity default resources") + { + ErrorString(Format("File access: %s is not linear Reading: %d Seek position: %d", fileName.c_str(), position, m_LastFileAccessPosition)); + } + + m_LastFileAccessPosition = position + size; + #endif +} + + +bool FileCacherRead::FreeSingleCache () +{ + unsigned lowestTimeStamp = -1; + CacheBlocks::iterator lowestCacheBlock = m_CacheBlocks.end (); + + CacheBlocks::iterator i; + for (i=m_CacheBlocks.begin ();i != m_CacheBlocks.end ();i++) + { + if (i->second.lockCount == 0) + { + if (i->second.timeStamp < lowestTimeStamp) + { + lowestTimeStamp = i->second.timeStamp; + lowestCacheBlock = i; + } + } + } + + if (lowestCacheBlock != m_CacheBlocks.end ()) + { + CacheBlock& block = lowestCacheBlock->second; + UNITY_FREE(kMemFile,block.data); + m_CacheBlocks.erase (lowestCacheBlock); + return true; + } + else + return false; +} + +FileCacherRead::CacheBlock& FileCacherRead::AllocateCacheBlock (int block) +{ + AssertIf (m_CacheBlocks.count (block)); + CacheBlock cacheBlock; + cacheBlock.block = block; + cacheBlock.lockCount = 0; + cacheBlock.timeStamp = 0; + cacheBlock.data = (UInt8*)UNITY_MALLOC(MemLabelId(kMemFileId, m_RootHeader),m_CacheSize); + m_CacheBlocks[block] = cacheBlock; + return m_CacheBlocks[block]; +} + +void FileCacherRead::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + CacheBlock* newCacheBlock; + CacheBlocks::iterator i = m_CacheBlocks.find (block); + if (i != m_CacheBlocks.end ()) + newCacheBlock = &i->second; + else + { + // Make room for a new cache block + if (m_CacheBlocks.size () >= m_MaxCacheCount) + FreeSingleCache (); + + newCacheBlock = &AllocateCacheBlock (block); + ReadCacheBlock (*newCacheBlock); + } + AssertIf (newCacheBlock->block != block); + + newCacheBlock->timeStamp = ++m_TimeStamp; + newCacheBlock->lockCount++; + + *startPos = newCacheBlock->data; + *endPos = newCacheBlock->data + min<int> (m_FileSize - block * GetCacheSize (), GetCacheSize ()); +} + +void FileCacherRead::UnlockCacheBlock (int block) +{ + AssertIf (!m_CacheBlocks.count (block)); + CacheBlock& cacheBlock = m_CacheBlocks.find (block)->second; + cacheBlock.lockCount--; + if (cacheBlock.lockCount == 0) + { + // LockCacheBlock sometimes locks more caches than m_MaxCacheCount + // Thus we should Free them as soon as they become unlocked + if (m_CacheBlocks.size () > m_MaxCacheCount) + FreeSingleCache (); + } +} + +std::string FileCacherRead::GetPathName() const +{ + return m_Path; +} + +#if SUPPORT_SERIALIZE_WRITE +FileCacherWrite::FileCacherWrite () +{ + m_Success = true; + m_Block = -1; + m_Locked = false; + m_CacheSize = 0; + m_DataCache = NULL; +} + +void FileCacherWrite::InitWriteFile (const std::string& pathName, size_t cacheSize) +{ + m_Path = PathToAbsolutePath(pathName); + m_Success = true; + + m_File.Open(m_Path, File::kWritePermission); + // file we're writing to always is a temporary, non-indexable file + SetFileFlags(m_Path, kAllFileFlags, kFileFlagDontIndex|kFileFlagTemporary); + + m_Block = -1; + m_Locked = false; + m_CacheSize = cacheSize; + m_DataCache = (UInt8*)UNITY_MALLOC (kMemFile, m_CacheSize); +} + +bool FileCacherWrite::CompleteWriting (size_t size) +{ + Assert(m_Block != -1); + + size_t remainingData = size - (m_Block * m_CacheSize); + Assert(remainingData <= m_CacheSize); + + m_Success &= m_File.Write(m_Block * m_CacheSize, m_DataCache, remainingData); + + return m_Success; +} + +bool FileCacherWrite::WriteHeaderAndCloseFile (void* data, size_t position, size_t size) +{ + Assert(position == 0); + if (size != 0) + m_Success &= m_File.Write(position, data, size); + m_Success &= m_File.Close(); + + return m_Success; +} + + +FileCacherWrite::~FileCacherWrite() +{ + if (m_DataCache) + { + UNITY_FREE (kMemFile, m_DataCache); + m_DataCache = NULL; + } + + m_File.Close(); +} + +void FileCacherWrite::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + AssertIf (block == -1); + Assert (block == m_Block || m_Block+1 == block ); + Assert (!m_Locked); + + if (m_Block != block) + { + AssertIf (m_Locked != 0); + + if (m_Block != -1) + m_Success &= m_File.Write(m_DataCache, m_CacheSize); + + m_Block = block; + } + + *startPos = m_DataCache; + *endPos = m_DataCache + m_CacheSize; + m_Locked++; +} + +void FileCacherWrite::UnlockCacheBlock (int block) +{ + Assert (block == m_Block); + Assert (m_Locked); + + m_Locked = false; +} + + +std::string FileCacherWrite::GetPathName() const +{ + return m_Path; +} + +#endif // SUPPORT_SERIALIZE_WRITE + + +MemoryCacherReadBlocks::MemoryCacherReadBlocks (UInt8** blocks, int size, size_t cacheBlockSize) +: m_CacheBlockSize(cacheBlockSize) +, m_Memory(blocks) +, m_FileSize(size) +{ +} + + +MemoryCacherReadBlocks::~MemoryCacherReadBlocks () +{ +} + +void MemoryCacherReadBlocks::LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) +{ + /// VERIFY OUT OF BOUNDS!!! ??? + AssertIf(block > m_FileSize / m_CacheBlockSize); + *startPos = m_Memory[block]; + *endPos = *startPos + min<int> (GetFileLength () - block * m_CacheBlockSize, m_CacheBlockSize); +} + +void MemoryCacherReadBlocks::DirectRead (void* data, size_t position, size_t size) +{ + ReadFileCache(*this, data, position, size); +} + +void ReadFileCache (CacheReaderBase& cacher, void* data, size_t position, size_t size) +{ + UInt8 *cacheStart, *cacheEnd; + UInt8 *from, *fromClamped, *to, *toClamped; + + int block = position / cacher.GetCacheSize (); + int lastBlock = (position + size - 1) / cacher.GetCacheSize (); + + while (block <= lastBlock) + { + cacher.LockCacheBlock (block, &cacheStart, &cacheEnd); + + // copy data from oldblock and unlock + from = cacheStart + (position - block * cacher.GetCacheSize ()); + fromClamped = clamp (from, cacheStart, cacheEnd); + to = cacheStart + (position + size - block * cacher.GetCacheSize ()); + toClamped = clamp<UInt8*> (to, cacheStart, cacheEnd); + memcpy ((UInt8*)data + (fromClamped - from), fromClamped, toClamped - fromClamped); + + cacher.UnlockCacheBlock (block); + block++; + } +} diff --git a/Runtime/Serialize/FileCache.h b/Runtime/Serialize/FileCache.h new file mode 100644 index 0000000..6705b44 --- /dev/null +++ b/Runtime/Serialize/FileCache.h @@ -0,0 +1,360 @@ +#ifndef FILECACHE_H +#define FILECACHE_H + +#include "Runtime/Utilities/LogAssert.h" +#include "Configuration/UnityConfigure.h" +#include "Runtime/Utilities/File.h" +#include <list> +#include <vector> +#include <map> +#include <deque> +#include "Runtime/Allocator/MemoryMacros.h" +#include "Runtime/Utilities/dynamic_array.h" + +#define DEBUG_LINEAR_FILE_ACCESS 0 + +using std::max; +using std::min; +using std::list; + +class EXPORT_COREMODULE CacheReaderBase +{ +public: + + enum + { + kImmediatePriority = 0, + kPreloadPriority = 1, + kLoadPriorityCount = 2 + }; + + enum + { + kUnloadPriority = 0, + kPreloadedPriority = 1, + kRequiredPriority = 2, + kUnloadPriorityCount = 3 + }; + + virtual ~CacheReaderBase () = 0; + + virtual void DirectRead (void* data, size_t position, size_t size) = 0; + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) = 0; + virtual void UnlockCacheBlock (int block) = 0; + + virtual size_t GetCacheSize () const = 0; + virtual std::string GetPathName() const = 0; + virtual size_t GetFileLength () const = 0; + virtual UInt8* GetAddressOfMemory() { ErrorString("GetAddressOfMemory called on CacheReaderBase which does not support it"); return NULL; } +}; + +class CacheWriterBase +{ +public: + virtual ~CacheWriterBase () = 0; + + virtual bool CompleteWriting (size_t size) = 0; + virtual bool WriteHeaderAndCloseFile (void* /*data*/, size_t /*position*/, size_t /*size*/) { AssertString("Only used for writing serialized files"); return false; } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) = 0; + virtual void UnlockCacheBlock (int block) = 0; + + virtual size_t GetCacheSize () const = 0; + virtual std::string GetPathName() const = 0; + virtual size_t GetFileLength () const = 0; + virtual UInt8* GetAddressOfMemory() { ErrorString("GetAddressOfMemory called on CacheWriterBase which does not support it"); return NULL; } +}; + +class MemoryCacheReader : public CacheReaderBase +{ +protected: + enum + { + kCacheSize = 256 + }; + + dynamic_array<UInt8>& m_Memory; + SInt32 m_LockCount; + +public: + MemoryCacheReader (dynamic_array<UInt8>& mem) : m_Memory (mem), m_LockCount(0) { } + virtual ~MemoryCacheReader () { AssertIf (m_LockCount != 0); } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) + { + *startPos = m_Memory.size() > block * kCacheSize ? &m_Memory[block * kCacheSize] : NULL; + *endPos = *startPos + min<int> (GetFileLength () - block * kCacheSize, kCacheSize); + m_LockCount++; + } + + virtual void DirectRead (void* data, size_t position, size_t size) + { + AssertIf (m_Memory.size () - position < size); + memcpy (data, &m_Memory[position], size); + } + + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + + virtual size_t GetFileLength () const { return m_Memory.size (); } + virtual size_t GetCacheSize () const { return kCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } + virtual UInt8* GetAddressOfMemory() { return &m_Memory[0]; } +}; + +class MemoryCacheWriter : public CacheWriterBase +{ +protected: + enum + { + kCacheSize = 256 + }; + + dynamic_array<UInt8>& m_Memory; + SInt32 m_LockCount; + +public: + MemoryCacheWriter (dynamic_array<UInt8>& mem) : m_Memory (mem), m_LockCount(0) { } + virtual ~MemoryCacheWriter () { AssertIf (m_LockCount != 0); } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) + { + m_Memory.resize_uninitialized (max<int> ((block + 1) * kCacheSize, m_Memory.size ()), true); + *startPos = &m_Memory[block * kCacheSize]; + *endPos = *startPos + kCacheSize; + + m_LockCount++; + } + + virtual bool CompleteWriting (size_t size) { m_Memory.resize_uninitialized (size); m_Memory.shrink_to_fit(); return true; } + + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + + virtual size_t GetFileLength () const { return m_Memory.size (); } + virtual size_t GetCacheSize () const { return kCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } +}; + +enum +{ + kBlockCacherCacheSize = 256 +}; + +class BlockMemoryCacheWriter : public CacheWriterBase +{ +protected: + + enum + { + kNumBlockReservations = 256 + }; + + size_t m_Size; + SInt32 m_LockCount; + MemLabelId m_AllocLabel; + + // It is possible to use the custom allocator for this index as well -- however, + // using the tracking linear tempory allocator is most efficient, when deallocating + // in the exact opposite order of allocating, which can only be guaranteed, when all allocations + // are in our control. + typedef UNITY_VECTOR(kMemFile, UInt8*) BlockVector; + BlockVector m_Blocks; + + public: + + BlockMemoryCacheWriter (MemLabelId label) + : m_AllocLabel(label) + , m_Blocks() + { + m_Blocks.reserve(kNumBlockReservations); + m_Size = 0; + m_LockCount = 0; + } + + ~BlockMemoryCacheWriter () { + AssertIf (m_LockCount != 0); + for(BlockVector::reverse_iterator i = m_Blocks.rbegin(); i != m_Blocks.rend(); i++) + UNITY_FREE(m_AllocLabel, *i); + } + + void ResizeBlocks (int newBlockSize) + { + int oldBlockSize = m_Blocks.size(); + + for(int block = oldBlockSize-1; block>=newBlockSize; block--) + UNITY_FREE(m_AllocLabel, m_Blocks[block]); + + if(m_Blocks.capacity() < newBlockSize) + m_Blocks.reserve(m_Blocks.capacity() * 2); + + m_Blocks.resize(newBlockSize); + + for(int block = oldBlockSize; block<newBlockSize; block++) + m_Blocks[block] = (UInt8*)UNITY_MALLOC(m_AllocLabel, kBlockCacherCacheSize); + } + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos) + { + ResizeBlocks (max<int>(block+1, m_Blocks.size())); + *startPos = m_Blocks[block]; + *endPos = *startPos + kBlockCacherCacheSize; + m_LockCount++; + } + + virtual bool CompleteWriting (size_t size) + { + m_Size = size; + ResizeBlocks(m_Size/kBlockCacherCacheSize + 1); + return true; + } + virtual void UnlockCacheBlock (int /*block*/) { m_LockCount--; } + virtual size_t GetFileLength () const { return m_Size; } + virtual size_t GetCacheSize () const { return kBlockCacherCacheSize; } + virtual std::string GetPathName() const { return "MemoryStream"; } + + // Expose, so the internal data can be used with MemoryCacherReadBlocks (see below) + UInt8** GetCacheBlocks () { return m_Blocks.empty () ? NULL : &*m_Blocks.begin (); } +}; + +#if SUPPORT_SERIALIZE_WRITE + +/// Used by SerializedFile to write to disk. +/// Currently it doesn't allow any seeking that is, you can only write blocks in consecutive order +class FileCacherWrite : public CacheWriterBase +{ +public: + FileCacherWrite (); + void InitWriteFile (const std::string& pathName, size_t cacheSize); + + virtual ~FileCacherWrite (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int block); + + virtual bool WriteHeaderAndCloseFile (void* data, size_t position, size_t size); + + virtual bool CompleteWriting (size_t size); + + virtual size_t GetCacheSize () const { return m_CacheSize; } + virtual std::string GetPathName() const; + virtual size_t GetFileLength () const { AssertString("Dont use"); return 0; } + +private: + void WriteBlock (int block); + + int m_Block; + UInt8* m_DataCache; + size_t m_CacheSize; + + File m_File; + bool m_Success; + bool m_Locked; + std::string m_Path; +}; + +#endif // SUPPORT_SERIALIZE_WRITE + + +/// Used by SerializedFile to read from disk +class FileCacherRead : public CacheReaderBase +{ +public: + FileCacherRead (const std::string& pathName, size_t cacheSize, size_t cacheCount); + ~FileCacherRead (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int block); + + virtual void DirectRead (void* data, size_t position, size_t size); + + virtual size_t GetFileLength () const { return m_FileSize; } + virtual size_t GetCacheSize () const { return m_CacheSize; } + virtual std::string GetPathName() const; + +private: + + // Finishes all reading, deletes all caches + void Flush (); + + struct CacheBlock + { + UInt8* data; + int block; + int lockCount; + unsigned timeStamp; + }; + + // Reads the cacheBlock from disk. + void ReadCacheBlock (CacheBlock& cacheBlock); + + /// Allocates an cache block at block + CacheBlock& AllocateCacheBlock (int block); + + // Frees the cache block with the smallest timestamp that is not locked. + // Returns whether or not a cache block could be freed. + bool FreeSingleCache (); + + typedef UNITY_MAP(kMemFile, int, CacheBlock) CacheBlocks; + CacheBlocks m_CacheBlocks; + + size_t m_CacheSize; + size_t m_MaxCacheCount; + size_t m_FileSize; + UInt32 m_TimeStamp; + std::string m_Path; + File m_File; + ProfilerAllocationHeader* m_RootHeader; + friend struct OpenFilesCache; + + #if DEBUG_LINEAR_FILE_ACCESS + int m_LastFileAccessPosition; + #endif + +}; + +enum +{ + // Evil: Must match the in CompressedFileStreamMemory.h, UncompressedFileStreamMemory.h" + kCacheBlockSize = 1024 * 100 +}; + +/// Used by SerializedFile to read from disk +class MemoryCacherReadBlocks : public CacheReaderBase +{ +public: + + MemoryCacherReadBlocks (UInt8** blocks, int size, size_t cacheBlockSize); + ~MemoryCacherReadBlocks (); + + virtual void LockCacheBlock (int block, UInt8** startPos, UInt8** endPos); + virtual void UnlockCacheBlock (int /*block*/) {} + + virtual void DirectRead (void* data, size_t position, size_t size); + + virtual size_t GetFileLength () const { return m_FileSize; } + + virtual size_t GetCacheSize () const { return m_CacheBlockSize; } + + virtual std::string GetPathName() const { return "none"; } + + virtual UInt8* GetAddressOfMemory() + { + return m_Memory[0]; + } + +private: + + // Finishes all reading, deletes all caches + void Flush (); + + UInt8** m_Memory; + size_t m_FileSize; + size_t m_CacheBlockSize; +}; + +void ReadFileCache (CacheReaderBase& cacher, void* data, size_t position, size_t size); +#if UNITY_EDITOR +void ForceCloseAllOpenFileCaches (); +#endif + +#endif + diff --git a/Runtime/Serialize/FloatStringConversion.cpp b/Runtime/Serialize/FloatStringConversion.cpp new file mode 100644 index 0000000..ce057d3 --- /dev/null +++ b/Runtime/Serialize/FloatStringConversion.cpp @@ -0,0 +1,80 @@ +#include "UnityPrefix.h" + +#if UNITY_EDITOR +// This are the definitions of std::numeric_limits<>::max_digits10, which we cannot use +// because it is only in the C++11 standard. +const int kMaxFloatDigits = std::floor(std::numeric_limits<float>::digits * 3010.0/10000.0 + 2); +const int kMaxDoubleDigits = std::floor(std::numeric_limits<double>::digits * 3010.0/10000.0 + 2); + +#include "External/gdtoa/gdtoa.h" + +bool FloatToStringAccurate (float f, char* buffer, size_t maximumSize) +{ + return g_ffmt (buffer, (float*)&f, kMaxFloatDigits, maximumSize) != NULL; +} + +bool DoubleToStringAccurate (double f, char* buffer, size_t maximumSize) +{ + return g_dfmt (buffer, (double*)&f, kMaxDoubleDigits, maximumSize) != NULL; +} + +bool FloatToStringAccurate (float f, UnityStr& output) +{ + char buf[64]; + if (FloatToStringAccurate (f, buf, 64)) + { + output = buf; + return true; + } + else + return false; +} + +bool DoubleToStringAccurate (double f, UnityStr& output) +{ + char buf[64]; + if (DoubleToStringAccurate (f, buf, 64)) + { + output = buf; + return true; + } + else + return false; +} + +float StringToFloatAccurate (const char* buffer) +{ + return strtof (buffer, NULL); +} + +double StringToDoubleAccurate (const char* buffer) +{ + return strtod (buffer, NULL); +} + +#if ENABLE_UNIT_TESTS +#include "../../External/UnitTest++/src/UnitTest++.h" + +SUITE (FloatStringConversionTests) +{ +TEST(FloatToStringConversion_AccurateWorks) +{ + // Make sure no locale is used + CHECK_EQUAL (0.0F, StringToFloatAccurate("0,5")); + + UnityStr buf; + + FloatToStringAccurate(1.0F, buf); + CHECK_EQUAL ("1", buf); + + CHECK_EQUAL (1.0F, StringToFloatAccurate("1.0")); + + + FloatToStringAccurate(1.5F, buf); + CHECK_EQUAL ("1.5", buf); + CHECK_EQUAL (1.5F, StringToFloatAccurate("1.5")); +} +} +#endif + +#endif diff --git a/Runtime/Serialize/FloatStringConversion.h b/Runtime/Serialize/FloatStringConversion.h new file mode 100644 index 0000000..c93d143 --- /dev/null +++ b/Runtime/Serialize/FloatStringConversion.h @@ -0,0 +1,16 @@ +#if UNITY_EDITOR + +/// Converts float/double to and from strings. + +/// Binary exact float<-> string conversion functions. +/// Supports the full range of all float values, including nan, inf and denormalized values. + +bool FloatToStringAccurate (float f, char* buffer, size_t maximumSize); +bool DoubleToStringAccurate (double f, char* buffer, size_t maximumSize); + +bool FloatToStringAccurate (float f, UnityStr& output); +bool DoubleToStringAccurate (double f, UnityStr& output); + +float StringToFloatAccurate (const char* buffer); +double StringToDoubleAccurate (const char* buffer); +#endif
\ No newline at end of file diff --git a/Runtime/Serialize/IterateTypeTree.h b/Runtime/Serialize/IterateTypeTree.h new file mode 100644 index 0000000..4532b9b --- /dev/null +++ b/Runtime/Serialize/IterateTypeTree.h @@ -0,0 +1,155 @@ +#ifndef ITERATETYPETREE_H +#define ITERATETYPETREE_H + +#include "TypeTree.h" +#include "SerializeTraits.h" +#include "TransferUtility.h" + + +/* Iterate typetree is used to process serialized data in arbitrary ways. + +struct IterateTypeTreeFunctor +{ + // return true if you want to recurse into the function + bool operator () (const TypeTree& typeTree, dynamic_array<UInt8>& data, int bytePosition) + { + + } +} + +TypeTree typeTree; +dynamic_array<UInt8> data +// Create typetree and data +GenerateTypeTree(object); +WriteObjectToVector(object, &data); + +// Modify data +IterateTypeTreeFunctor func; +IterateTypeTree (typeTree, data, func); + +ReadObjectFromVector(&object, data, typeTree); +object.CheckConsistency (); +object.AwakeFromLoad (false); +object.SetDirty (); + +*/ + +inline SInt32 ExtractPPtrInstanceID (const UInt8* data) +{ + return *reinterpret_cast<const SInt32*> (data); +} + +inline SInt32 ExtractPPtrInstanceID (const dynamic_array<UInt8>& data, int bytePosition) +{ + return ExtractPPtrInstanceID(&data[bytePosition]); +} + +inline void SetPPtrInstanceID (SInt32 instanceID, dynamic_array<UInt8>& data, int bytePosition) +{ + *reinterpret_cast<SInt32*> (&data[bytePosition]) = instanceID; +} + +inline bool IsTypeTreePPtr (const TypeTree& typeTree) +{ + return typeTree.m_Type.find ("PPtr<") == 0; +} + +inline bool IsTypeTreeString (const TypeTree& typeTree) +{ + return typeTree.m_Type == "string" && typeTree.m_Children.size() == 1 && typeTree.m_Children.back().m_IsArray; +} + +inline bool IsTypeTreePPtrArray (const TypeTree& typeTree) +{ + return typeTree.m_IsArray && typeTree.m_Children.back().m_Type.find ("PPtr<") == 0; +} + +inline bool IsTypeTreeArraySize (const TypeTree& typeTree) +{ + return typeTree.m_Father != NULL && typeTree.m_Father->m_IsArray && &typeTree.m_Father->m_Children.front() == &typeTree; +} + +inline bool IsTypeTreeArrayElement (const TypeTree& typeTree) +{ + return typeTree.m_Father != NULL && typeTree.m_Father->m_IsArray && &typeTree.m_Father->m_Children.back() == &typeTree; +} + +inline bool IsTypeTreeArrayOrArrayContainer (const TypeTree& typeTree) +{ + return typeTree.m_IsArray || (typeTree.m_Children.size() == 1 && typeTree.m_Children.back().m_IsArray); +} + +inline bool IsTypeTreeArray (const TypeTree& typeTree) +{ + return typeTree.m_IsArray; +} + +inline int ExtractArraySize (const UInt8* data) +{ + return *reinterpret_cast<const SInt32*> (data); +} + +inline void SetArraySize (UInt8* data, SInt32 size) +{ + *reinterpret_cast<SInt32*> (data) = size; +} + +inline int ExtractArraySize (const dynamic_array<UInt8>& data, int bytePosition) +{ + return *reinterpret_cast<const SInt32*> (&data[bytePosition]); +} + + +inline UInt32 Align4_Iterate (UInt32 size) +{ + UInt32 value = ((size + 3) >> 2) << 2; + return value; +} +#if UNITY_EDITOR + +template<class Functor> +void IterateTypeTree (const TypeTree& typeTree, dynamic_array<UInt8>& data, Functor& functor) +{ + int bytePosition = 0; + IterateTypeTree (typeTree, data, &bytePosition, functor); +} +template<class Functor> +void IterateTypeTree (const TypeTree& typeTree, dynamic_array<UInt8>& data, int* bytePosition, Functor& functor) +{ + if (functor (typeTree, data, *bytePosition)) + { + if (typeTree.m_IsArray) + { + // First child in an array is the size + // Second child is the homogenous type of the array + AssertIf (typeTree.m_Children.front ().m_Type != SerializeTraits<SInt32>::GetTypeString (NULL) || typeTree.m_Children.front ().m_Name != "size" || typeTree.m_Children.size () != 2); + + functor (typeTree.m_Children.front (), data, *bytePosition); + + SInt32 arraySize, i; + arraySize = *reinterpret_cast<SInt32*> (&data[*bytePosition]); + *bytePosition += sizeof (arraySize); + + for (i=0;i<arraySize;i++) + IterateTypeTree (typeTree.m_Children.back (), data, bytePosition, functor); + } + else + { + TypeTree::TypeTreeList::const_iterator i; + for (i = typeTree.m_Children.begin (); i != typeTree.m_Children.end ();++i) + IterateTypeTree (*i, data, bytePosition, functor); + } + + if (typeTree.IsBasicDataType ()) + *bytePosition += typeTree.m_ByteSize; + + if (typeTree.m_MetaFlag & kAlignBytesFlag) + *bytePosition = Align4_Iterate (*bytePosition); + } + else + { + WalkTypeTree(typeTree, data.begin (), bytePosition); + } +} +#endif +#endif diff --git a/Runtime/Serialize/LoadProgress.h b/Runtime/Serialize/LoadProgress.h new file mode 100644 index 0000000..d8fdabb --- /dev/null +++ b/Runtime/Serialize/LoadProgress.h @@ -0,0 +1,24 @@ +#ifndef LOAD_PROGRESS_H +#define LOAD_PROGRESS_H + +class LoadProgress +{ + volatile float* progressIndicator; + float progressInterval; + +public: + float totalItems; + float processedItems; + + LoadProgress(unsigned total, float interval, float* indicator) : processedItems(0), totalItems(total), progressIndicator(indicator), progressInterval (interval) {} + + void ItemProcessed (int count = 1) + { + processedItems = std::min (totalItems, processedItems + count); + + if (progressIndicator) + *progressIndicator = totalItems == 0 ? 1.0f : progressInterval * processedItems / totalItems; + } + +}; +#endif diff --git a/Runtime/Serialize/PathNamePersistentManager.cpp b/Runtime/Serialize/PathNamePersistentManager.cpp new file mode 100644 index 0000000..81022e7 --- /dev/null +++ b/Runtime/Serialize/PathNamePersistentManager.cpp @@ -0,0 +1,51 @@ +#include "UnityPrefix.h" +#include "PathNamePersistentManager.h" +#include "Runtime/Utilities/Word.h" + +using namespace std; + +int PathNamePersistentManager::InsertPathNameInternal (const string& pathname, bool create) +{ + SET_ALLOC_OWNER(NULL); + AssertIf (!pathname.empty () && (pathname[0] == '/' || pathname[0] == '\\')); + + string lowerCasePathName = ToLower (pathname); + + PathToStreamID::iterator found = m_PathToStreamID.find (lowerCasePathName); + if (found != m_PathToStreamID.end()) + return found->second; + + if (create) + { + m_PathToStreamID.insert (make_pair (lowerCasePathName, m_PathNames.size ())); + m_PathNames.push_back (pathname); + AddStream (); + return m_PathNames.size () - 1; + } + else + return -1; +} + +int PathNamePersistentManager::InsertFileIdentifierInternal (FileIdentifier file, bool create) +{ + return InsertPathNameInternal(file.pathName, create); +} + +FileIdentifier PathNamePersistentManager::PathIDToFileIdentifierInternal (int pathID) +{ + AssertIf (pathID < 0 || pathID >= m_PathNames.size ()); + FileIdentifier f; + f.pathName = m_PathNames[pathID]; + return f; +} + +string PathNamePersistentManager::PathIDToPathNameInternal (int pathID) +{ + AssertIf (pathID < 0 || pathID >= m_PathNames.size ()); + return m_PathNames[pathID]; +} + +void InitPathNamePersistentManager() +{ + UNITY_NEW_AS_ROOT( PathNamePersistentManager(0), kMemManager, "PathNameManager", ""); +} diff --git a/Runtime/Serialize/PathNamePersistentManager.h b/Runtime/Serialize/PathNamePersistentManager.h new file mode 100644 index 0000000..11c3767 --- /dev/null +++ b/Runtime/Serialize/PathNamePersistentManager.h @@ -0,0 +1,31 @@ +#ifndef PATHNAMEPERSISTENTMANAGER_H +#define PATHNAMEPERSISTENTMANAGER_H + +#include "PersistentManager.h" +#include "SerializedFile.h" + +class PathNamePersistentManager : public PersistentManager +{ + typedef map<string, SInt32> PathToStreamID; + PathToStreamID m_PathToStreamID; // Contains lower case pathnames + vector<string> m_PathNames;// Contains pathnames as they were given + + public: + + PathNamePersistentManager (int options, int cacheCount = 2) + : PersistentManager (options, cacheCount) {} + + protected: + + virtual int InsertPathNameInternal (const std::string& pathname, bool create); + virtual int InsertFileIdentifierInternal (FileIdentifier file, bool create); + + virtual string PathIDToPathNameInternal (int pathID); + + virtual FileIdentifier PathIDToFileIdentifierInternal (int pathID); +}; + +void InitPathNamePersistentManager(); + + +#endif diff --git a/Runtime/Serialize/PersistentManager.cpp b/Runtime/Serialize/PersistentManager.cpp new file mode 100644 index 0000000..3f596f4 --- /dev/null +++ b/Runtime/Serialize/PersistentManager.cpp @@ -0,0 +1,2291 @@ +#include "UnityPrefix.h" +#include "PersistentManager.h" +#include "Runtime/BaseClasses/BaseObject.h" +#include "Runtime/Utilities/FileUtilities.h" +#include "SerializedFile.h" +#include "Runtime/Utilities/LogAssert.h" +#include "Remapper.h" +#include "Runtime/Utilities/Word.h" +#include "Runtime/Profiler/Profiler.h" +#include "Runtime/Utilities/File.h" +#include "AwakeFromLoadQueue.h" + +#define DEBUG_THREAD_LOAD 0 +#define DEBUG_THREAD_LOAD_LONG_ACTIVATE !UNITY_RELEASE +#define DEBUG_MAINTHREAD_LOADING 0 + + +#if DEBUG_THREAD_LOAD +#define printf_debug_thread printf_console +#else +#define printf_debug_thread +#endif + +#include "Runtime/Threads/ProfilerMutex.h" +#if ENABLE_PROFILER +#include "Runtime/Profiler/MemoryProfiler.h" +#endif +PROFILER_INFORMATION(gMakeObjectUnpersistentProfiler, "Loading.MakeObjectUnpersistent", kProfilerLoading) +PROFILER_INFORMATION(gMakeObjectPersistentProfiler, "Loading.MakeObjectUnpersistent", kProfilerLoading) +PROFILER_INFORMATION(gAwakeFromLoadManager, "Loading.AwakeFromLoad", kProfilerLoading) +PROFILER_INFORMATION(gIDRemappingProfiler, "Loading.IDRemapping", kProfilerLoading) +PROFILER_INFORMATION(gWriteFileProfiler, "Loading.WriteFile", kProfilerLoading) +PROFILER_INFORMATION(gFindInActivationQueueProfiler, "Loading.FindInThreadedActivationQueue", kProfilerLoading) +PROFILER_INFORMATION(gReadObjectProfiler, "Loading.ReadObject", kProfilerLoading) +PROFILER_INFORMATION(gLoadFileProfiler, "Loading.LoadFile", kProfilerLoading) +PROFILER_INFORMATION(gIsObjectAvailable, "Loading.IsObjectAvailaable", kProfilerLoading) +PROFILER_INFORMATION(gLoadStreamNameSpaceProfiler, "Loading.LoadFileHeaders", kProfilerLoading) +PROFILER_INFORMATION(gLoadLockPersistentManager, "Loading.LockPersistentManager", kProfilerLoading) +PROFILER_INFORMATION(gLoadFromActivationQueueStall, "Loading.LoadFromActivationQueue stalled [wait for loading operation to finish]", kProfilerLoading) +// @TODO: Write test for cross references between monobehaviours in prefab and in scene! + + +static PersistentManager* gPersistentManager = NULL; +static PersistentManager::InOrderDeleteCallbackFunction* gInOrderDeleteCallback = NULL; +static PersistentManager::SafeBinaryReadCallbackFunction* gSafeBinaryReadCallback = NULL; + +static const char* kSerializedFileArea = "SerializedFile"; +static const char* kRemapperAllocArea = "PersistentManager.Remapper"; + +#if UNITY_EDITOR || SUPPORT_RESOURCE_IMAGE_LOADING +static const char* kResourceImageExtensions[] = { "resG", "res", "resS" }; +#endif + +double GetTimeSinceStartup (); + +using namespace std; + +bool PersistentManager::InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + return m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier); +} + +int PersistentManager::SerializedObjectIdentifierToInstanceID (const SerializedObjectIdentifier& identifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + return m_Remapper->GetOrGenerateMemoryID (identifier); +} + + +LocalIdentifierInFileType PersistentManager::GetLocalFileID(SInt32 instanceID) +{ + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier (instanceID, identifier); + return identifier.localIdentifierInFile; +} + +SInt32 PersistentManager::GetInstanceIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = InsertPathNameInternal (path, true); + identifier.localIdentifierInFile = localIdentifierInFile; + return m_Remapper->GetOrGenerateMemoryID (identifier); +} + +int PersistentManager::GetClassIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = InsertPathNameInternal (path, true); + identifier.localIdentifierInFile = localIdentifierInFile; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return -1; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return -1; + + return stream->GetClassID (identifier.localIdentifierInFile); +} + +static void CleanupStream (StreamNameSpace& stream) +{ + SerializedFile* oldFile = stream.stream; + stream.stream = NULL; + + UNITY_DELETE (oldFile, kMemSerialization); +} + +int PersistentManager::GetSerializedClassID (int instanceID) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier)) + return -1; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return -1; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return -1; + + return stream->GetClassID (identifier.localIdentifierInFile); +} + +void PersistentManager::GetAllFileIDs (const string& pathName, vector<LocalIdentifierInFileType>* objects) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, true); + SerializedFile* stream = GetSerializedFileInternal (serializedFileIndex); + if (stream == NULL) + return; + + stream->GetAllFileIDs (objects); +} + +bool PersistentManager::RemoveObjectsFromPath (const std::string& pathName) +{ +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + ASSERT_RUNNING_ON_MAIN_THREAD + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SInt32 serializedFileIndex = InsertPathNameInternal (pathName, false); + if (serializedFileIndex == -1) + return false; + + vector<SInt32> temp; + m_Remapper->RemoveCompletePathID(serializedFileIndex, temp); + + return true; +} + +void PersistentManager::MakeObjectUnpersistent (int memoryID, UnpersistMode mode) +{ + PROFILER_AUTO_THREAD_SAFE(gMakeObjectUnpersistentProfiler, NULL); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + ASSERT_RUNNING_ON_MAIN_THREAD + + Object* o = Object::IDToPointer (memoryID); + if (o && !o->IsPersistent ()) + { +// #if DEBUGMODE && !UNITY_RELEASE +// AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, GetMainThreadID(), "PersistentManager.MakeObjectUnpersistent"); +// AssertIf (m_Remapper->GetPathID (memoryID) != -1); +// #endif + return; + } + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + if (mode == kDestroyFromFile) + DestroyFromFileInternal (memoryID); + + m_Remapper->Remove (memoryID); + + if (o) + o->SetIsPersistent (false); +} + +#if UNITY_EDITOR +void PersistentManager::MakeObjectPersistent (int heapID, const string& pathName) +{ + MakeObjectPersistentAtFileID (heapID, 0, pathName); +} + +void PersistentManager::MakeObjectPersistentAtFileID (int heapID, LocalIdentifierInFileType fileID, const string& pathName) +{ + MakeObjectsPersistent (&heapID, &fileID, 1, pathName); +} + +void PersistentManager::MakeObjectsPersistent (const int* heapIDs, LocalIdentifierInFileType* fileIDs, int size, const string& pathName, int options) +{ + PROFILER_AUTO_THREAD_SAFE(gMakeObjectPersistentProfiler, NULL); + + AssertIf(!m_AllowLoadingFromDisk); + AssertIf(!Thread::EqualsCurrentThreadID(GetMainThreadID())); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + AssertIf(pathName.empty()); + SInt32 globalNameSpace = InsertPathNameInternal (pathName, true); + StreamNameSpace* streamNameSpace = NULL; + + for (int i=0;i<size;i++) + { + int heapID = heapIDs[i]; + LocalIdentifierInFileType fileID = fileIDs[i]; + + Object* o = Object::IDToPointer (heapID); + + if ((options & kMakePersistentDontRequireToBeLoadedAndDontUnpersist) == 0) + { + // Making an object that is not in memory persistent + if (o == NULL) + { + ErrorString("Make Objects Persistent failed because the object can not be loaded"); + continue; + } + + // Make Object unpersistent first + if (o->IsPersistent ()) + { + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier(heapID, identifier); + AssertIf (identifier.serializedFileIndex == -1); + + // Return if the file and serializedFileIndex is not going to change + if (globalNameSpace == identifier.serializedFileIndex) + { + if (fileID == 0 || identifier.localIdentifierInFile == fileID) + continue; + } + + MakeObjectUnpersistent (heapID, kDestroyFromFile); + } + } + + if (streamNameSpace == NULL) + streamNameSpace = &GetStreamNameSpaceInternal (globalNameSpace); + + // Allocate an fileID for this object in the File + if (fileID == 0) + { + fileID = streamNameSpace->highestID; + if (streamNameSpace->stream) + fileID = max (streamNameSpace->highestID, streamNameSpace->stream->GetHighestID ()); + fileID++; + } + streamNameSpace->highestID = max (streamNameSpace->highestID, fileID); + + SerializedObjectIdentifier identifier; + identifier.serializedFileIndex = globalNameSpace; + identifier.localIdentifierInFile = fileID; + m_Remapper->SetupRemapping (heapID, identifier); + fileIDs[i] = fileID; + + if (o) + { + + AssertIf (o->TestHideFlag (Object::kDontSave) && (options & kAllowDontSaveObjectsToBePersistent) == 0); + o->SetIsPersistent (true); + o->SetDirty (); + } + } +} +#endif + + +void PersistentManager::LocalSerializedObjectIdentifierToInstanceIDInternal (const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID) +{ + int activeNameSpace = m_ActiveNameSpace.top(); + LocalSerializedObjectIdentifierToInstanceIDInternal (activeNameSpace, localIdentifier, outInstanceID); +} + +void PersistentManager::LocalSerializedObjectIdentifierToInstanceIDInternal (int activeNameSpace, const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + + LocalIdentifierInFileType localIdentifierInFile = localIdentifier.localIdentifierInFile; + int localSerializedFileIndex = localIdentifier.localSerializedFileIndex; + + if (localIdentifierInFile == 0) + { + outInstanceID = 0; + return; + } + + AssertIf (localSerializedFileIndex == -1); + + int globalFileIndex; + if (localSerializedFileIndex == 0) + globalFileIndex = activeNameSpace; + else + { + AssertIf (m_Streams[activeNameSpace].stream == NULL); + + AssertIf(activeNameSpace >= m_LocalToGlobalNameSpace.size() || activeNameSpace < 0); + + IDRemap::iterator found = m_LocalToGlobalNameSpace[activeNameSpace].find (localSerializedFileIndex); + + if (found != m_LocalToGlobalNameSpace[activeNameSpace].end ()) + { + globalFileIndex = found->second; + } + else + { + AssertString ("illegal LocalPathID in persistentmanager"); + outInstanceID = 0; + return; + } + } + + SerializedObjectIdentifier globalIdentifier; + globalIdentifier.serializedFileIndex = globalFileIndex; + globalIdentifier.localIdentifierInFile = localIdentifierInFile; + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + ApplyInstanceIDRemap(globalIdentifier); + #endif + + outInstanceID = m_Remapper->GetOrGenerateMemoryID (globalIdentifier); +} + +#if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD +void PersistentManager::ApplyInstanceIDRemap(SerializedObjectIdentifier& id) +{ + InstanceIDRemap::iterator foundIDRemap = m_InstanceIDRemap.find(id); + if (foundIDRemap != m_InstanceIDRemap.end()) + id = foundIDRemap->second; +} +#endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + +LocalSerializedObjectIdentifier PersistentManager::GlobalToLocalSerializedFileIndexInternal (const SerializedObjectIdentifier& globalIdentifier) +{ + LocalIdentifierInFileType localIdentifierInFile = globalIdentifier.localIdentifierInFile; + int localSerializedFileIndex; + + // Remap globalPathID to localPathID + int activeNameSpace = m_ActiveNameSpace.top (); + + IDRemap& globalToLocalNameSpace = m_GlobalToLocalNameSpace[activeNameSpace]; + IDRemap& localToGlobalNameSpace = m_LocalToGlobalNameSpace[activeNameSpace]; + + IDRemap::iterator found = globalToLocalNameSpace.find (globalIdentifier.serializedFileIndex); + if (found == globalToLocalNameSpace.end ()) + { + SET_ALLOC_OWNER(NULL); + AssertIf (activeNameSpace >= m_Streams.size()); + AssertIf (m_Streams[activeNameSpace].stream == NULL); + SerializedFile& serialize = *m_Streams[activeNameSpace].stream; + + serialize.AddExternalRef (PathIDToFileIdentifierInternal (globalIdentifier.serializedFileIndex)); + + localSerializedFileIndex = serialize.GetExternalRefs ().size (); + globalToLocalNameSpace[globalIdentifier.serializedFileIndex] = localSerializedFileIndex; + localToGlobalNameSpace[localSerializedFileIndex] = globalIdentifier.serializedFileIndex; + } + else + localSerializedFileIndex = found->second; + + // Setup local identifier + LocalSerializedObjectIdentifier localIdentifier; + + localIdentifier.localSerializedFileIndex = localSerializedFileIndex; + localIdentifier.localIdentifierInFile = localIdentifierInFile; + + return localIdentifier; +} + +void PersistentManager::InstanceIDToLocalSerializedObjectIdentifierInternal (SInt32 instanceID, LocalSerializedObjectIdentifier& localIdentifier) +{ + PROFILER_AUTO_THREAD_SAFE(gIDRemappingProfiler, NULL); + + AssertIf (m_ActiveNameSpace.empty ()); + if (instanceID == 0) + { + localIdentifier.localSerializedFileIndex = 0; + localIdentifier.localIdentifierInFile = 0; + return; + } + + SerializedObjectIdentifier globalIdentifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier (instanceID, globalIdentifier)) + { + localIdentifier.localSerializedFileIndex = 0; + localIdentifier.localIdentifierInFile = 0; + return; + } + + localIdentifier = GlobalToLocalSerializedFileIndexInternal(globalIdentifier); +} + +bool PersistentManager::IsInstanceIDFromCurrentFileInternal (SInt32 instanceID) +{ + if (instanceID == 0) + return false; + + SerializedObjectIdentifier globalIdentifier; + + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier (instanceID, globalIdentifier)) + return false; + + int activeNameSpace = m_ActiveNameSpace.top (); + return globalIdentifier.serializedFileIndex == activeNameSpace; +} + +#if UNITY_EDITOR + +int PersistentManager::GetSerializedFileIndexFromPath (const std::string& path) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + return InsertPathNameInternal (path, true); +} + +bool PersistentManager::TestNeedWriteFile (const string& pathName, const std::set<int>* cachedDirtyPathsHint) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = InsertPathNameInternal (pathName, false); + return TestNeedWriteFileInternal(serializedFileIndex, cachedDirtyPathsHint); +} + +bool PersistentManager::TestNeedWriteFile (int globalFileIndex, const std::set<int>* cachedDirtyPathsHint) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + return TestNeedWriteFileInternal(globalFileIndex, cachedDirtyPathsHint); +} + +bool PersistentManager::TestNeedWriteFileInternal (int globalFileIndex, const std::set<int>* cachedDirtyPathsHint) +{ + if (globalFileIndex == -1) + return false; + + SerializedFile* stream = m_Streams[globalFileIndex].stream; + + bool isFileDirty = stream != NULL && stream->IsFileDirty (); + + // Something was deleted from the file. Must write it! + if (isFileDirty) + return true; + + // Use Dirty path indices to quickly determine if a file needs writing + if (cachedDirtyPathsHint != NULL) + return cachedDirtyPathsHint->count (globalFileIndex); + + Object* o; + // Find out if file needs to write to disk + // - Get all loaded objects that have registered themselves for being at that file + set<SInt32> loadedWriteObjects; + m_Remapper->GetAllLoadedObjectsAtPath (globalFileIndex, &loadedWriteObjects); + for (set<SInt32>::iterator i=loadedWriteObjects.begin ();i != loadedWriteObjects.end ();i++) + { + o = Object::IDToPointer (*i); + if (o && o->IsPersistent () && o->IsPersistentDirty ()) + return true; + } + + return false; +} + +void PersistentManager::CleanupStreamAndNameSpaceMapping (unsigned serializedFileIndex) +{ + // Unload the file any way + // This saves memory - especially when reimporting lots of assets like when rebuilding the library + CleanupStream(m_Streams[serializedFileIndex]); + + m_GlobalToLocalNameSpace[serializedFileIndex].clear (); + m_LocalToGlobalNameSpace[serializedFileIndex].clear (); +} + +static bool InitTempWriteFile (FileCacherWrite& writer, const std::string& path, unsigned cacheSize) +{ + string tempWriteFileName = GenerateUniquePathSafe (path); + if (tempWriteFileName.empty()) + return false; + + writer.InitWriteFile(path, cacheSize); + + return true; +} + +int PersistentManager::WriteFile (const std::string& path, BuildTargetSelection target, int options) +{ + PROFILER_AUTO_THREAD_SAFE(gWriteFileProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex; + serializedFileIndex = InsertPathNameInternal(path, false); + if (serializedFileIndex == -1) + return kNoError; + + bool needsWrite = TestNeedWriteFile(serializedFileIndex); + + // Early out + if (!needsWrite) + { + // @TODO: THIS SHOULD NOT BE HACKED IN HERE. Make test coverage against increased leaking then remove this and call CleanupStream explicitly. + CleanupStreamAndNameSpaceMapping(serializedFileIndex); + return kNoError; + } + + set<SInt32> writeObjects; + if (options & kDontReadObjectsFromDiskBeforeWriting) + { + GetLoadedInstanceIDsAtPath (path, &writeObjects); + Assert(!writeObjects.empty()); + } + else + { + // Load all writeobjects into memory + // (dont use LoadFileCompletely, since that reads all objects + // even those that might have been changed in memory) + GetInstanceIDsAtPath (path, &writeObjects); + } + + vector<WriteData> writeData; + + for (set<SInt32>::iterator i=writeObjects.begin ();i != writeObjects.end ();i++) + { + SInt32 instanceID = *i; + + // Force load object from disk. + Object* o = dynamic_instanceID_cast<Object*> (instanceID); + + if (o == NULL) + continue; + + #if UNITY_EDITOR + // Disable text serialization for terrain data. Just too much data, you'd never want to merge. + int cid = o->GetClassID(); + if (IsClassNonTextSerialized(cid)) + options &= ~kAllowTextSerialization; + #endif + + AssertIf (o != NULL && !o->IsPersistent ()); + + SerializedObjectIdentifier identifier; + m_Remapper->InstanceIDToSerializedObjectIdentifier(instanceID, identifier); + + Assert (identifier.serializedFileIndex == serializedFileIndex); + + DebugAssertIf (!o->IsPersistent ()); + DebugAssertIf (m_Remapper->GetSerializedFileIndex (instanceID) != serializedFileIndex); + DebugAssertIf (!m_Remapper->IsSetup (identifier)); + + writeData.push_back(WriteData (identifier.localIdentifierInFile, instanceID, BuildUsageTag())); + } + + sort(writeData.begin(), writeData.end()); + + int result = WriteFileInternal(path, serializedFileIndex, &writeData[0], writeData.size(), NULL, target, options); + if (result != kNoError && options & kAllowTextSerialization) + // Try binary serialization as a fallback. + result = WriteFileInternal(path, serializedFileIndex, &writeData[0], writeData.size(), NULL, target, options &~kAllowTextSerialization); + + return result; +} + +int PersistentManager::WriteFileInternal (const string& path, int serializedFileIndex, const WriteData* writeObjectData, int size, VerifyWriteObjectCallback* verifyCallback, BuildTargetSelection target, int options) +{ + //printf_console("Writing file %s\n", pathName.c_str()); + + // Create writing tools + CachedWriter writer; + + FileCacherWrite serializedFileWriter; + FileCacherWrite resourceImageWriters[kNbResourceImages]; + if (!InitTempWriteFile (serializedFileWriter, "Temp/tempFile", kCacheSize)) + return kFileCouldNotBeWritten; + writer.InitWrite(serializedFileWriter); + + if (options & kBuildResourceImage) + { + for (int i=0;i<kNbResourceImages;i++) + { + string path = AppendPathNameExtension("Temp/tempFile", kResourceImageExtensions[i]); + if (!InitTempWriteFile (resourceImageWriters[i], path, kCacheSize)) + return kFileCouldNotBeWritten; + writer.InitResourceImage((ActiveResourceImage)i, resourceImageWriters[i]); + } + } + + // Cleanup old stream and mapping + CleanupStreamAndNameSpaceMapping(serializedFileIndex); + + // Setup global to self namespace mapping + m_GlobalToLocalNameSpace[serializedFileIndex][serializedFileIndex] = 0; + m_LocalToGlobalNameSpace[serializedFileIndex][0] = serializedFileIndex; + + // Create writable stream + //@TODO: Object name might want to be + SerializedFile* tempSerialize = UNITY_NEW_AS_ROOT(SerializedFile, kMemSerialization, kSerializedFileArea, ""); + #if ENABLE_MEM_PROFILER + tempSerialize->SetDebugPath(PathIDToPathNameInternal(serializedFileIndex)); + GetMemoryProfiler()->SetRootAllocationObjectName(tempSerialize, tempSerialize->GetDebugPath().c_str()); + #endif + + tempSerialize->InitializeWrite (writer, target, options); + m_Streams[serializedFileIndex].stream = tempSerialize; + + m_ActiveNameSpace.push (serializedFileIndex); + + bool writeSuccess = true; + // Write Objects in fileID order + for (int i=0;i<size;i++) + { + LocalIdentifierInFileType localIdentifierInFile = writeObjectData[i].localIdentifierInFile; + SInt32 instanceID = writeObjectData[i].instanceID; + + SerializedObjectIdentifier identifier (serializedFileIndex, localIdentifierInFile); + + bool shouldUnloadImmediately = false; + + Object* o = Object::IDToPointer (instanceID);; + if (o == NULL) + { + if (options & kLoadAndUnloadAssetsDuringBuild) + { + o = dynamic_instanceID_cast<Object*> (instanceID); + shouldUnloadImmediately = true; + } + + // Object can not be loaded, don't write it + if (o == NULL) + { + continue; + } + } + + if (verifyCallback != NULL && !verifyCallback (o, target.platform)) + writeSuccess = false; + + tempSerialize->WriteObject (*o, localIdentifierInFile, writeObjectData[i].buildUsage); + o->ClearPersistentDirty (); + + if (shouldUnloadImmediately) + UnloadObject(o); + } + + m_ActiveNameSpace.pop(); + + writeSuccess = writeSuccess && tempSerialize->FinishWriting() && !tempSerialize->HasErrors(); + + // Delete temp stream + if (m_Streams[serializedFileIndex].stream != tempSerialize) + { + writeSuccess = false; + UNITY_DELETE (tempSerialize, kMemSerialization); + tempSerialize = NULL; + } + + // Delete mappings + CleanupStreamAndNameSpaceMapping(serializedFileIndex); + + if (!writeSuccess) + { +// ErrorString ("Writing file: " + path + " failed. The temporary file " + serializedFileWriter.GetPathName() + " couldn't be written."); + return kFileCouldNotBeWritten; + } + + // Atomically move the serialized file into the target location + string actualNewPathName = RemapToAbsolutePath (path); + if (!MoveReplaceFile (serializedFileWriter.GetPathName(), actualNewPathName)) + { + ErrorString ("File " + path + " couldn't be written. Because moving " + serializedFileWriter.GetPathName() + " to " + actualNewPathName + " failed."); + return kFileCouldNotBeWritten; + } + SetFileFlags(actualNewPathName, kFileFlagTemporary, 0); + + + if (options & kBuildResourceImage) + { + // Move the resource images into the target location + for (int i=0;i<kNbResourceImages;i++) + { + string targetPath = AppendPathNameExtension(actualNewPathName, kResourceImageExtensions[i]); + + ::DeleteFile(targetPath); + + string tempWriteFileName = resourceImageWriters[i].GetPathName(); + + if (GetFileLength(tempWriteFileName) > 0) + { + if (!MoveReplaceFile (tempWriteFileName, targetPath)) + { + ErrorString ("File " + path + " couldn't be written. Because moving " + tempWriteFileName + " to " + actualNewPathName + " failed."); + return kFileCouldNotBeWritten; + } + SetFileFlags(targetPath, kFileFlagTemporary, 0); + } + } + } + + return kNoError; +} + +#endif + +string PersistentManager::GetPathName (SInt32 memoryID) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SInt32 serializedFileIndex = m_Remapper->GetSerializedFileIndex (memoryID); + if (serializedFileIndex == -1) + return string (); + else + return PathIDToPathNameInternal (serializedFileIndex); +} + +void PersistentManager::RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex (const ThreadedAwakeData& awake) +{ + // Register instance ID first and then unlock integration mutex so there is no chance of an object being loaded twice + AssertIf(!awake.completedThreadAwake); + if (awake.object != NULL) + { + Object::RegisterInstanceID(awake.object); + + m_IntegrationMutex.Unlock(); + + AwakeFromLoadMode mode = (AwakeFromLoadMode)(kDidLoadFromDisk | kDidLoadThreaded); + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject(*awake.object, awake.oldType, mode, awake.checkConsistency, gSafeBinaryReadCallback); +} + else + { + m_IntegrationMutex.Unlock(); + } +} + + +#if THREADED_LOADING +void PersistentManager::AllowIntegrationWithTimeoutAndWait () +{ + m_IntegrationMutex.Lock(); + m_AllowIntegrateThreadedObjectsWithTimeout = true; + m_IntegrationMutex.Unlock(); + + // Wait until the integration thread has integrated all assets + while (true) + { + m_IntegrationMutex.Lock(); + + if (m_ThreadedObjectActivationQueue.empty()) + { + m_IntegrationMutex.Unlock(); + break; + } + else + { + m_IntegrationMutex.Unlock(); + } + + Thread::Sleep(0.1F); + } + + m_IntegrationMutex.Lock(); + m_AllowIntegrateThreadedObjectsWithTimeout = false; + m_IntegrationMutex.Unlock(); +} +#else +void PersistentManager::AllowIntegrationWithTimeoutAndWait () +{ + IntegrateAllThreadedObjects(); +} +#endif + +#if DEBUG_THREAD_LOAD +int gDependencyCounter = 0; +int gDependencyCounterCost = 0; +int gDependencyCounterCostActivation = 0; +int gDependencyCounterCostNonActivation = 0; +int gDependencyCounterCostNotFound = 0; +#endif + +Object* PersistentManager::LoadFromActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + ASSERT_RUNNING_ON_MAIN_THREAD + + LOCK_MUTEX(m_IntegrationMutex, gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator imap = m_ThreadedObjectActivationMap.find(heapID); + + if (imap != m_ThreadedObjectActivationMap.end()) + { + ThreadedObjectActivationQueue::iterator i = imap->second; + if (!i->completedThreadAwake) + { + ErrorString("Internal thread activation error. Activating object that has not been fully thread loaded."); + m_IntegrationMutex.Unlock(); + + return NULL; + } + + ThreadedAwakeData data = *i; + m_ThreadedObjectActivationQueue.erase(i); + m_ThreadedObjectActivationMap.erase(imap); + #if DEBUG_THREAD_LOAD +// printf_debug_thread("Activating dependency %d / %d [%d]\n", ++gDependencyCounter, gDependencyCounterCost, thisCost); +// if (thisCost > 300) +// { +// printf_debug_thread(""); +// } + #endif + + RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex(data); + + return data.object; + } + + #if DEBUG_THREAD_LOAD +// if (thisCost > 300) +// { +// printf_debug_thread("Expensive load from activation queue and not found [%d]\n", thisCost); +// } + #endif + + m_IntegrationMutex.Unlock(); + + return NULL; +} + +// GetFromActivationQueue is called a lot, so this function must be very efficient. +// In one of the games called Harvest, m_ThreadedObjectActivationQueue contained 70000 items, and this function was called 5678 times +// so linear searching was ubber slow!!! +Object* PersistentManager::GetFromActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_IntegrationMutex,gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator item = m_ThreadedObjectActivationMap.find(heapID); + if (item != m_ThreadedObjectActivationMap.end()) + { + return item->second->object; + } + + AssertIf(m_OnDemandThreadLoadedObjects.count(heapID)); + + return NULL; +} + +bool PersistentManager::FindInActivationQueue (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gFindInActivationQueueProfiler, NULL); + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_IntegrationMutex, gLoadFromActivationQueueStall); + + ThreadedObjectActivationMap::iterator item = m_ThreadedObjectActivationMap.find(heapID); + if (item != m_ThreadedObjectActivationMap.end()) + { + return true; + } + + if (m_OnDemandThreadLoadedObjects.count(heapID)) + return true; + + return false; +} + +bool PersistentManager::HasThreadedObjectsToIntegrate () +{ + Mutex::AutoLock lock (m_IntegrationMutex); + return !m_ThreadedObjectActivationQueue.empty (); +} + +///@TODO: Prevent Object destruction during AwakeFromLoad!!! + +void PersistentManager::IntegrateThreadedObjects (float timeout) +{ + // Early out if there is nothing to be integrated. + if (!m_AllowIntegrateThreadedObjectsWithTimeout) + return; + + if (!m_IntegrationMutex.TryLock()) + { + //printf_debug_thread("\nINTEGRATION THREAD IS LOCKED\n"); + return; + } + + if (m_ThreadedObjectActivationQueue.empty() || !m_AllowIntegrateThreadedObjectsWithTimeout) + { + m_IntegrationMutex.Unlock(); + return; + } + + double startTime = GetTimeSinceStartup(); + #if DEBUG_THREAD_LOAD + int integratedObjects = 0; + #endif + + while (true) + { + ThreadedAwakeData awake = m_ThreadedObjectActivationQueue.front(); + + if (!awake.completedThreadAwake) + { + ErrorString("Stalling Integration because ThreadAwake for object is not completed"); + break; + } + + #if DEBUG_THREAD_LOAD + integratedObjects++; + #endif + m_ThreadedObjectActivationMap.erase(awake.instanceID); + m_ThreadedObjectActivationQueue.pop_front(); + + #if DEBUG_THREAD_LOAD_LONG_ACTIVATE + double time = GetTimeSinceStartup(); + #endif + + RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex(awake); + + #if DEBUG_THREAD_LOAD_LONG_ACTIVATE + time = GetTimeSinceStartup() - time; + time *= 1000.0F; + if (time > 30.0F) + { + printf_console("Long thread activation time (%d ms) for object %s (%s)\n", (int)time, awake.object->GetName(), awake.object->GetClassName().c_str()); + } + #endif + + if (!m_AllowIntegrateThreadedObjectsWithTimeout || !m_IntegrationMutex.TryLock()) + { + #if DEBUG_THREAD_LOAD + startTime = (GetTimeSinceStartup() - startTime) * 1000.0F; + printf_debug_thread("Integrate Threaded Objects ms: %f %d\n\n", (float)startTime, integratedObjects); + #endif + return; + } + + double delta = (GetTimeSinceStartup() - startTime); + if (m_ThreadedObjectActivationQueue.empty() || delta > timeout) + break; + } + + #if DEBUG_THREAD_LOAD + startTime = (GetTimeSinceStartup() - startTime) * 1000.0F; + printf_debug_thread("Integrate Threaded Objects ms: %f %d\n\n", (float)startTime, integratedObjects); + #endif + + m_IntegrationMutex.Unlock(); +} + +void PersistentManager::IntegrateAllThreadedObjects () +{ + AwakeFromLoadQueue awakeQueue (kMemTempAlloc); + + PrepareAllThreadedObjectsStep1 (awakeQueue); + awakeQueue.RegisterObjectInstanceIDs(); + IntegrateAllThreadedObjectsStep2 (awakeQueue); +} + +void PersistentManager::PrepareAllThreadedObjectsStep1 (AwakeFromLoadQueue& awakeQueue) +{ + AQUIRE_AUTOLOCK (m_IntegrationMutex, gLoadFromActivationQueueStall); + + // Add to AwakeFromLoadQueue - this will take care of ensuring sort order + awakeQueue.Reserve(m_ThreadedObjectActivationQueue.size()); + + ThreadedObjectActivationQueue::iterator end = m_ThreadedObjectActivationQueue.end(); + for (ThreadedObjectActivationQueue::iterator i=m_ThreadedObjectActivationQueue.begin();i != end;++i ) + { + ThreadedAwakeData& awake = *i; + + if (awake.object) + { + Assert (awake.completedThreadAwake); + awakeQueue.Add(*awake.object, awake.oldType, awake.checkConsistency); + } + } + m_ThreadedObjectActivationQueue.clear(); + m_ThreadedObjectActivationMap.clear(); +} + +void PersistentManager::IntegrateAllThreadedObjectsStep2 (AwakeFromLoadQueue& awakeQueue) +{ + Assert(m_ThreadedObjectActivationQueue.empty() && m_ThreadedObjectActivationMap.empty()); + + // Invoke AwakeFromLoadQueue in sorted order + AwakeFromLoadMode awakeMode = (AwakeFromLoadMode)(kDidLoadFromDisk | kDidLoadThreaded); + + awakeQueue.PersistentManagerAwakeFromLoad(awakeMode, gSafeBinaryReadCallback); + + Assert(m_ThreadedObjectActivationQueue.empty() && m_ThreadedObjectActivationMap.empty()); +} + + +bool PersistentManager::ReloadFromDisk (Object* obj) +{ + PROFILER_AUTO_THREAD_SAFE(gReadObjectProfiler, obj); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(obj->GetInstanceID(), identifier)) + { + ErrorString("Trying to reload asset from disk that is not stored on disk"); + return false; + } + + SerializedFile* stream = GetSerializedFileInternal(identifier.serializedFileIndex); + if (stream == NULL) + { + ErrorString("Trying to reload asset but can't find object on disk"); + return false; + } + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + stream->ReadObject (identifier.localIdentifierInFile, obj->GetInstanceID(), kCreateObjectDefault, true, &oldType, &didTypeTreeChange, &obj); + m_ActiveNameSpace.pop (); + + // Awake the object + if (obj) + { + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (*obj, oldType, kDidLoadFromDisk, didTypeTreeChange, gSafeBinaryReadCallback); + } + + return true; +} + +Object* PersistentManager::ReadObject (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gReadObjectProfiler, NULL); + +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + #if !UNITY_EDITOR + AssertIf(heapID < 0); + #endif + + #if DEBUG_MAINTHREAD_LOADING + double time = GetTimeSinceStartup (); + #endif + + ASSERT_RUNNING_ON_MAIN_THREAD + + LOCK_MUTEX(m_Mutex, gLoadLockPersistentManager); + + Object* o = LoadFromActivationQueue(heapID); + if (o != NULL) +{ + m_Mutex.Unlock(); + return o; + } + + // Find and load the right stream + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + { + m_Mutex.Unlock(); + return NULL; + } + +#if DEBUGMODE + AssertIf(!m_IsLoadingSceneFile && StrICmp(GetPathNameExtension(PathIDToPathNameInternal(identifier.serializedFileIndex)),"unity") == 0); +#endif + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + { + #if DEBUG_MAINTHREAD_LOADING + LogString(Format("--- Loading from main thread failed loading stream %f", (GetTimeSinceStartup () - time) * 1000.0F)); + #endif + + m_Mutex.Unlock(); + return NULL; +} + + AssertIf(Object::IDToPointer (heapID) != NULL); + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + o = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectDefault, true, &oldType, &didTypeTreeChange, &o); + m_ActiveNameSpace.pop (); + + // Awake the object + if (o) + { + AwakeFromLoadQueue::PersistentManagerAwakeSingleObject (*o, oldType, kDidLoadFromDisk, didTypeTreeChange, gSafeBinaryReadCallback); + } + +// printf_console ("Read %d %s (%s)\n", heapID, o ? o->GetName () : "<null>", o ? o->GetClassName ().c_str() : ""); + + m_Mutex.Unlock(); + + #if DEBUG_MAINTHREAD_LOADING + if (o) + { + LogString(Format("--- Loading from main thread %s (%s) %f ms", o->GetName(), o->GetClassName().c_str(), (GetTimeSinceStartup () - time) * 1000.0F)); + } + else + { + LogString(Format("--- Loading from main thread (NULL) %f ms", (GetTimeSinceStartup () - time) * 1000.0F)); + } + #endif + + return o; +} + +void PersistentManager::SetupThreadActivationQueueObject (ThreadedAwakeData& awakeData, TypeTree* oldType, bool didTypeTreeChange) +{ + Object* obj = awakeData.object; + if (obj != NULL) +{ + awakeData.oldType = oldType; + awakeData.checkConsistency = didTypeTreeChange; + obj->AwakeFromLoadThreaded(); + } + awakeData.completedThreadAwake = true; +} + +ThreadedAwakeData* PersistentManager::CreateThreadActivationQueueEntry (SInt32 instanceID) +{ + DebugAssertIf(Object::IDToPointerThreadSafe(instanceID) != NULL); + + ThreadedAwakeData awake; + awake.instanceID = instanceID; + awake.checkConsistency = false; + awake.completedThreadAwake = false; + awake.oldType = NULL; + awake.object = NULL; + + ThreadedAwakeData* result; + m_IntegrationMutex.Lock(); + + m_ThreadedObjectActivationQueue.push_back(awake); + result = &m_ThreadedObjectActivationQueue.back(); + m_ThreadedObjectActivationMap[instanceID] = --m_ThreadedObjectActivationQueue.end(); + + m_IntegrationMutex.Unlock(); + DebugAssertIf(Object::IDToPointerThreadSafe(instanceID) != NULL); + return result; +} + +void PersistentManager::CheckInstanceIDsLoaded (SInt32* heapIDs, int size) +{ + m_IntegrationMutex.Lock(); + + set<SInt32> ids; + + // Search through threaded object activation queue and activate the object if necessary. + ThreadedObjectActivationQueue::iterator i, end; + end = m_ThreadedObjectActivationQueue.end(); + for (i=m_ThreadedObjectActivationQueue.begin();i != end;i++) + ids.insert(i->instanceID); + + for (int j=0;j<size;j++) + { + if (ids.count(heapIDs[j])) + heapIDs[j] = 0; + } + + m_IntegrationMutex.Unlock(); + + // Check which objects are already loaded all at once to lock object creation only once for a short amount of time + // Since we have locked persistentmanager already no objects can be loaded in the mean time + LockObjectCreation(); + Object::CheckInstanceIDsLoaded(heapIDs, size); + UnlockObjectCreation(); +} + +Object* PersistentManager::ReadObjectThreaded (int heapID) +{ +#if DEBUGMODE + AssertIf(!m_AllowLoadingFromDisk); +#endif + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + Object* o = GetFromActivationQueue(heapID); + if (o != NULL) + return o; + + // Find and load the right stream + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + return NULL; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + return NULL; +#if DEBUGMODE + AssertIf(!m_IsLoadingSceneFile && PathIDToPathNameInternal(identifier.serializedFileIndex).find(".unity") != string::npos); +#endif + AssertIf(Object::IDToPointerThreadSafe (heapID) != NULL); + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry(heapID); + + // Inject into m_OnDemandThreadLoadedObjects so we can track which objects have been read during a LoadFileCompletelyThreaded or LoadThreadedObjects. + // This prevents loading objects twice + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.insert(heapID); + m_IntegrationMutex.Unlock(); + + awakeData->object = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectFromNonMainThread, !m_Remapper->IsSceneID (heapID), &oldType, &didTypeTreeChange, &awakeData->object); + + m_ActiveNameSpace.pop (); + + o = awakeData->object; + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + + return o; +} + +void PersistentManager::LoadObjectsThreaded (SInt32* heapIDs, int size, LoadProgress* loadProgress) +{ + if (size == 0) + return; + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + vector<SInt32> heapIDsCopy; + heapIDsCopy.assign(heapIDs, heapIDs + size); + + // Check which objects are already loaded all at once to lock object creation only once for a short amount of time + // Since we have locked persistentmanager already no objects can be loaded in the mean time + CheckInstanceIDsLoaded(&heapIDsCopy[0], size); + + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + + for (int i=0;i<size;i++) + { + SInt32 heapID = heapIDsCopy[i]; + if (heapID == 0) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + m_IntegrationMutex.Lock(); + if (m_OnDemandThreadLoadedObjects.count(heapID)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + m_IntegrationMutex.Unlock(); + continue; + } + m_IntegrationMutex.Unlock(); + + DebugAssertIf(Object::IDToPointerThreadSafe(heapID) != NULL); + //DebugAssertIf(FindInActivationQueue(heapID)); + + // Find and load the right stream + SerializedObjectIdentifier identifier; + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + if (stream == NULL) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + // Find file id in stream and read the object + + m_ActiveNameSpace.push (identifier.serializedFileIndex); + TypeTree* oldType; + bool didTypeTreeChange; + + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry (heapID); + awakeData->object = NULL; + stream->ReadObject (identifier.localIdentifierInFile, heapID, kCreateObjectFromNonMainThread, true, &oldType, &didTypeTreeChange, &awakeData->object); + if (loadProgress) + loadProgress->ItemProcessed (); + + AssertIf (m_Remapper->IsSceneID (heapID)); + + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + + m_ActiveNameSpace.pop (); +} + + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.clear(); + m_IntegrationMutex.Unlock(); + + #if DEBUGMODE + m_IsLoadingSceneFile = false; + #endif + } + +int PersistentManager::LoadFileCompletelyThreaded (const std::string& pathname, LocalIdentifierInFileType* fileIDs, SInt32* instanceIDs, int size, bool loadScene, LoadProgress* loadProgress) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadFromActivationQueueStall); + + // Find Stream + int pathID = InsertPathNameInternal (pathname, true); + SerializedFile* stream = GetSerializedFileInternal (pathID); + if (stream == NULL) + return kFileCouldNotBeRead; + + AssertIf(fileIDs != NULL && size == -1); + AssertIf(instanceIDs != NULL && size == -1); + + // Get all file IDs we want to load and generate instance ids + vector<LocalIdentifierInFileType> fileIDsVector; + vector<SInt32> instanceIDsVector; + if (size == -1) + { + stream->GetAllFileIDs (&fileIDsVector); + fileIDs = &fileIDsVector[0]; + size = fileIDsVector.size(); + if (loadProgress) + loadProgress->totalItems += size; + instanceIDsVector.resize(size); + instanceIDs = &instanceIDsVector[0]; + } + + // In the editor we can not use preallocate ranges since fileID's might be completely arbitrary ranges + if (loadScene && !UNITY_EDITOR) + { + LocalIdentifierInFileType highestFileID = 0; + for (int i=0;i<size;i++) + { + AssertIf(fileIDs[i] < 0); + highestFileID = max(highestFileID, fileIDs[i]); + } + + m_Remapper->PreallocateIDs(highestFileID, pathID); + + for (int i=0;i<size;i++) + { + LocalIdentifierInFileType fileID = fileIDs[i]; + AssertIf(m_Remapper->IsSetup(SerializedObjectIdentifier(pathID, fileID))); + instanceIDs[i] = m_Remapper->m_ActivePreallocatedIDBase + fileID * 2; + } + + #if DEBUGMODE + //SHOULDN"T BE NEEDED!!!!!! - TAKE THIS OUT DEBUG ONLY!!!! + CheckInstanceIDsLoaded(&instanceIDs[0], size); + for (int i=0;i<size;i++) + { + AssertIf(instanceIDs[i] == 0); + } + #endif + } + else + { + for (int i=0;i<size;i++) + { + LocalIdentifierInFileType fileID = fileIDs[i]; + SInt32 heapID = m_Remapper->GetOrGenerateMemoryID (SerializedObjectIdentifier(pathID, fileID)); + + if (heapID == 0) + { + AssertString ("Loading an object that was made unpersistent but wasn't destroyed before reloading it"); + } + instanceIDs[i] = heapID; + } + // - Figure out which ones are already loaded + CheckInstanceIDsLoaded(&instanceIDs[0], size); + + #if UNITY_EDITOR + // Ugly hack to prevent IsPersistent to be on for scene objects. + // Recursive serialization is to be blamed for this. When that is fixed this can be removed. + if (loadScene) + { + m_Remapper->m_LoadingSceneInstanceIDs.assign_clear_duplicates(instanceIDs, instanceIDs + size); + } + #endif + } + + // Load all objects + m_ActiveNameSpace.push (pathID); + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + + for (int i=0;i<size;i++) + { + SInt32 heapID = instanceIDs[i]; + SInt32 fileID = fileIDs[i]; + + if (heapID == 0) + { + if (loadProgress) + loadProgress->ItemProcessed (); + continue; + } + + m_IntegrationMutex.Lock(); + if (m_OnDemandThreadLoadedObjects.count(heapID)) + { + if (loadProgress) + loadProgress->ItemProcessed (); + m_IntegrationMutex.Unlock(); + continue; + } + m_IntegrationMutex.Unlock(); + + DebugAssertIf(Object::IDToPointerThreadSafe (heapID) != NULL); + + TypeTree* oldType; + bool didTypeTreeChange; + ThreadedAwakeData* awakeData = CreateThreadActivationQueueEntry(heapID); + awakeData->object = NULL; + stream->ReadObject (fileID, heapID, kCreateObjectFromNonMainThread, !loadScene, &oldType, &didTypeTreeChange, &awakeData->object); + if (loadProgress) + loadProgress->ItemProcessed (); + + ////@TODO: NEED TO HANDLE DELETION OF OBJECTS WHEN THAT HAPPENS BETWEEN threaded loading and activation + SetupThreadActivationQueueObject(*awakeData, oldType, didTypeTreeChange); + } + + #if DEBUGMODE + m_IsLoadingSceneFile = false; + #endif + + m_ActiveNameSpace.pop (); + + m_IntegrationMutex.Lock(); + m_OnDemandThreadLoadedObjects.clear(); + m_IntegrationMutex.Unlock(); + if (loadScene) + { + #if UNITY_EDITOR + for (int i=0;i<size;i++) + m_Remapper->Remove (instanceIDs[i]); + m_Remapper->m_LoadingSceneInstanceIDs.clear(); + #else + m_Remapper->ClearPreallocateIDs(); + #endif + + } + + return kNoError; +} + +int PersistentManager::LoadFileCompletely (const string& path) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int result = GetPersistentManager().LoadFileCompletelyThreaded(path, NULL, NULL, -1, false, (LoadProgress*)NULL); + GetPersistentManager().IntegrateAllThreadedObjects (); + + return result; +} + + +#if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD +void PersistentManager::RemapInstanceIDOnLoad (const std::string& srcPath, LocalIdentifierInFileType srcLocalFileID, const std::string& dstPath, LocalIdentifierInFileType dstLocalFileID) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + SerializedObjectIdentifier src; + SerializedObjectIdentifier dst; + + src.serializedFileIndex = InsertPathNameInternal (srcPath, true); + src.localIdentifierInFile = srcLocalFileID; + Assert(src.serializedFileIndex != -1); + + // Destination path needs to be inserted (in unity_web_old case) + dst.serializedFileIndex = InsertPathNameInternal (dstPath, true); + dst.localIdentifierInFile = dstLocalFileID; + Assert(dst.serializedFileIndex != -1); + + m_InstanceIDRemap[src] = dst; +} +#endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + + +#if UNITY_EDITOR + +void PersistentManager::SuggestFileIDToHeapIDs (const string& pathname, map<LocalIdentifierInFileType, SInt32>& fileIDToHeapIDHint) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = InsertPathNameInternal (pathname, true); + + SerializedFile* stream = GetSerializedFileInternal (serializedFileIndex); + if (stream == NULL) + return; + + vector<LocalIdentifierInFileType> fileIDs; + stream->GetAllFileIDs (&fileIDs); + + for (vector<LocalIdentifierInFileType>::iterator i=fileIDs.begin ();i!=fileIDs.end ();++i) + { + LocalIdentifierInFileType fileID = *i; + SerializedObjectIdentifier identifier (serializedFileIndex, fileID); + + // fileIDToHeapIDHint is a hint which is used to keep the same fileID when entering/exiting playmode for example. + // It is only a hint and may not be fulfilled + if (!m_Remapper->IsSetup(identifier)) + { + if (fileIDToHeapIDHint.count(fileID)) + { + int suggestedHeapID = fileIDToHeapIDHint.find(fileID)->second; + if (Object::IDToPointer (suggestedHeapID) == NULL && !m_Remapper->IsHeapIDSetup(suggestedHeapID)) + { + m_Remapper->SetupRemapping(suggestedHeapID, identifier); + } + } + } + } +} +#endif + +void PersistentManager::GetLoadedInstanceIDsAtPath (const string& pathName, set<SInt32>* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, false); + if (serializedFileIndex != -1) + { + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllLoadedObjectsAtPath (serializedFileIndex, objects); + } +} + +void PersistentManager::GetPersistentInstanceIDsAtPath (const string& pathName, set<SInt32>* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int pathID = InsertPathNameInternal (pathName, true); + if (pathID == -1) + return; + + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllPersistentObjectsAtPath (pathID, objects); +} + +void PersistentManager::GetInstanceIDsAtPath (const string& pathName, set<SInt32>* objects) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (objects == NULL); + + int serializedFileIndex = InsertPathNameInternal (pathName, true); + if (serializedFileIndex == -1) + return; + + SerializedFile* serialize = GetSerializedFileInternal (serializedFileIndex); + if (serialize) + { + // Get all objects in the file + vector<LocalIdentifierInFileType> fileIDs; + serialize->GetAllFileIDs (&fileIDs); + for (vector<LocalIdentifierInFileType>::iterator i=fileIDs.begin ();i!=fileIDs.end ();++i) + { + SerializedObjectIdentifier identifier (serializedFileIndex, *i); + SInt32 memoryID = m_Remapper->GetOrGenerateMemoryID (identifier); + + if (memoryID != 0) + objects->insert (memoryID); + } + } + + // Get all objects that were made persistent but might not already be written to the file + m_Remapper->GetAllLoadedObjectsAtPath (serializedFileIndex, objects); +} + +void PersistentManager::GetInstanceIDsAtPath (const string& pathName, vector<SInt32>* objects) +{ + set<SInt32> temp; + GetInstanceIDsAtPath(pathName, &temp); + objects->assign(temp.begin(), temp.end()); +} + +int PersistentManager::CountInstanceIDsAtPath (const string& pathName) +{ + set<SInt32> objects; + GetInstanceIDsAtPath (pathName, &objects); + return objects.size (); +} + +bool PersistentManager::IsObjectAvailable (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gIsObjectAvailable, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + //////////////////////// DO WE WANT THIS!!!!!!!!!!!!!!!!!!!!!!!!!!! + if (FindInActivationQueue(heapID)) + return true; + + SerializedObjectIdentifier identifier; + + if (!m_Remapper->InstanceIDToSerializedObjectIdentifier(heapID, identifier)) + return false; + + SerializedFile* stream = GetSerializedFileInternal (identifier.serializedFileIndex); + // Stream can't be found + if (stream == NULL) + return false; + + if (!stream->IsAvailable (identifier.localIdentifierInFile)) + return false; + + // Check if the class can be produced + int classID = stream->GetClassID (identifier.localIdentifierInFile); + Object::RTTI* rtti = Object::ClassIDToRTTI (classID); + if (rtti && !rtti->isAbstract) + return true; + else + return false; +} + +bool PersistentManager::IsObjectAvailableDontCheckActualFile (int heapID) +{ + PROFILER_AUTO_THREAD_SAFE(gIsObjectAvailable, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int serializedFileIndex = m_Remapper->GetSerializedFileIndex (heapID); + if (serializedFileIndex == -1) + return false; + else + return true; +} + + +void PersistentManager::Lock() +{ + LOCK_MUTEX(m_Mutex, gLoadLockPersistentManager); +} + +void PersistentManager::Unlock() +{ + m_Mutex.Unlock(); +} + +void PersistentManager::DoneLoadingManagers() +{ + // We are done loading managers. Start instance IDs from a high constant value here, + // so new managers and built-in resources can be added without changed instanceIDs + // used by the content. + + //AssertIf(m_Remapper->GetHighestInUseHeapID() > 10000); + + if (m_Remapper->m_HighestMemoryID < 10000) + { + m_Remapper->m_HighestMemoryID = 10000; + } + + Object::DoneLoadingManagers(); +} + + +PersistentManager& GetPersistentManager () +{ + AssertIf (gPersistentManager == NULL); + return *gPersistentManager; +} + +PersistentManager* GetPersistentManagerPtr () +{ + return gPersistentManager; +} + +void CleanupPersistentManager() +{ + UNITY_DELETE( gPersistentManager, kMemManager); + gPersistentManager = NULL; +} + +PersistentManager::PersistentManager (int options, int cacheCount) +// list node size is base data type size + two node pointers. +: +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + m_ThreadedAwakeDataPool (false, "Remapper pool", sizeof (ThreadedAwakeData) + sizeof(void*)*2, 16 * 1024), + m_ThreadedAwakeDataPoolMap (false, "RemapperMap pool", sizeof (ThreadedObjectActivationQueue::iterator) + sizeof(void*)*5, 16 * 1024), + m_ThreadedObjectActivationQueue (m_ThreadedAwakeDataPool), + m_ThreadedObjectActivationMap (std::less<SInt32>(), m_ThreadedAwakeDataPoolMap), +#endif + m_AllowIntegrateThreadedObjectsWithTimeout (false) +{ + AssertIf (gPersistentManager); + gPersistentManager = this; + m_Options = options; + m_CacheCount = cacheCount; + m_Remapper = UNITY_NEW_AS_ROOT(Remapper (),kMemSerialization, kRemapperAllocArea, ""); + #if DEBUGMODE + m_IsLoadingSceneFile = true; + #endif + #if DEBUGMODE + m_PreventLoadingFromFile = -1; + m_AllowLoadingFromDisk = true; + #endif + + InitializeStdConverters (); +} + +PersistentManager::~PersistentManager () +{ + AssertIf(m_Mutex.IsLocked()); + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + AssertIf (!m_ActiveNameSpace.empty ()); + + for (StreamContainer::iterator i=m_Streams.begin ();i!=m_Streams.end ();++i) + CleanupStream(*i); + + UNITY_DELETE(m_Remapper, kMemSerialization); + CleanupStdConverters(); +} + +#if DEBUGMODE +void PersistentManager::SetDebugAssertLoadingFromFile (const std::string& path) +{ + if (path.empty()) + { + m_PreventLoadingFromFile = -1; + } + else + { + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + m_PreventLoadingFromFile = InsertPathNameInternal (path, false); + } +} +#endif + + +static bool IsPathBuiltinResourceFile (const std::string& pathName) +{ + return StrICmp(pathName, "library/unity default resources") == 0 || StrICmp(pathName, "library/unity_web_old") == 0 + || StrICmp(pathName, "library/unity editor resources") == 0; +} + +StreamNameSpace& PersistentManager::GetStreamNameSpaceInternal (int nameSpaceID) +{ +#if DEBUGMODE + AssertIf(m_PreventLoadingFromFile == nameSpaceID); +#endif + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + return nameSpace; + + #if SUPPORT_SERIALIZATION_FROM_DISK + + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + // Load Stream + string pathName = PathIDToPathNameInternal (nameSpaceID); + if (pathName.empty ()) + return nameSpace; + + // File not found + string absolutePath = RemapToAbsolutePath (pathName); + if (!IsFileCreated (absolutePath)) + { + #if !UNITY_EDITOR + AssertString("PersistentManager: Failed to open file at path: " + pathName); + #endif + return nameSpace; + } + + // Is Builtin resource file? + int options = 0; + if (IsPathBuiltinResourceFile(pathName)) + options |= kIsBuiltinResourcesFile; + + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile, kMemSerialization, kSerializedFileArea, ""); + SET_ALLOC_OWNER(nameSpace.stream); + #if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(pathName); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); + #endif + + // Resource image loading is only supported in the player! + ResourceImageGroup group; + #if SUPPORT_RESOURCE_IMAGE_LOADING + for (int i=0;i<kNbResourceImages;i++) + { + string resourceImagePath = AppendPathNameExtension(absolutePath, kResourceImageExtensions[i]); + if (IsFileCreated (resourceImagePath)) + group.resourceImages[i] = new ResourceImage(resourceImagePath, i == kStreamingResourceImage); + } + #endif + + if (!nameSpace.stream->InitializeRead (RemapToAbsolutePath (pathName), group, kCacheSize, m_CacheCount, options)) + { + CleanupStream(nameSpace); + return nameSpace; + } + + PostLoadStreamNameSpace(nameSpace, nameSpaceID); +#endif + + return m_Streams[nameSpaceID]; +} + +#if SUPPORT_SERIALIZATION_FROM_DISK +bool PersistentManager::LoadCachedFile (const std::string& pathName, const std::string actualAbsolutePath) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + ErrorString ("Tryng to load a stream which is already loaded."); + + // File not found + if (!IsFileCreated (actualAbsolutePath)) + return false; + + ResourceImageGroup group; + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + #if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(actualAbsolutePath); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); + #endif + if (!nameSpace.stream->InitializeRead (actualAbsolutePath, group, kCacheSize, m_CacheCount, kSerializeGameRelease)) + { + CleanupStream(nameSpace); + return false; + } + + nameSpace.stream->SetIsCachedFileStream(true); + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.insert(pathName); + + return true; +} +#endif + +bool PersistentManager::LoadMemoryBlockStream (const std::string& pathName, UInt8** data, int offset, int end, const char* url) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + AssertIf (nameSpace.stream); + + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + +#if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(url?url:pathName); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); +#endif + + int options = kSerializeGameRelease; + + // In NaCl & Flash, default resources are not loaded from disk, + // but also from a web stream. + // Need to set the proper flag here, so they don't get unloaded + // in Garbage Collection. + if (IsPathBuiltinResourceFile(pathName)) + options |= kIsBuiltinResourcesFile; + + if (!nameSpace.stream->InitializeMemoryBlocks (RemapToAbsolutePath(pathName), data, end, offset, options)) + { + CleanupStream(nameSpace); + return false; + } + + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + m_MemoryLoadedOrCachedPathsMutex.Lock(); + m_MemoryLoadedOrCachedPaths.insert(pathName); + m_MemoryLoadedOrCachedPathsMutex.Unlock(); + + return true; +} + + +#if SUPPORT_SERIALIZATION_FROM_DISK +bool PersistentManager::LoadExternalStream (const std::string& pathName, const std::string& absolutePath, int flags, int readOffset) +{ + PROFILER_AUTO_THREAD_SAFE(gLoadStreamNameSpaceProfiler, NULL); + + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, true); + if (nameSpaceID == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + + // Stream already loaded + if (nameSpace.stream) + return false; + + // File not found + if (!IsFileCreated (absolutePath)) + return false; + + ResourceImageGroup group; + nameSpace.stream = UNITY_NEW_AS_ROOT(SerializedFile,kMemSerialization, kSerializedFileArea, ""); + +#if ENABLE_MEM_PROFILER + nameSpace.stream->SetDebugPath(absolutePath); + GetMemoryProfiler()->SetRootAllocationObjectName(nameSpace.stream, nameSpace.stream->GetDebugPath().c_str()); +#endif + + if (!nameSpace.stream->InitializeRead (absolutePath, group, kCacheSize, m_CacheCount, flags, readOffset)) + { + CleanupStream(nameSpace); + return false; + } + + nameSpace.stream->SetIsCachedFileStream(true); + PostLoadStreamNameSpace(nameSpace, nameSpaceID); + + m_MemoryLoadedOrCachedPathsMutex.Lock(); + m_MemoryLoadedOrCachedPaths.insert(pathName); + m_MemoryLoadedOrCachedPathsMutex.Unlock(); + + return true; +} +#endif + +void PersistentManager::PostLoadStreamNameSpace (StreamNameSpace& nameSpace, int nameSpaceID) +{ + nameSpace.highestID = std::max (nameSpace.highestID, nameSpace.stream->GetHighestID ()); + SET_ALLOC_OWNER ( this ); + const dynamic_block_vector<FileIdentifier>& externalRefs = nameSpace.stream->GetExternalRefs (); + // Read all local pathnames and generate global<->localnamespace mapping + for (unsigned int i=0;i!=externalRefs.size ();i++) + { + int serializedFileIndex = InsertFileIdentifierInternal (externalRefs[i], true); + m_GlobalToLocalNameSpace[nameSpaceID][serializedFileIndex] = i + 1; + m_LocalToGlobalNameSpace[nameSpaceID][i + 1] = serializedFileIndex; + } + + // Setup global to self namespace mapping + m_GlobalToLocalNameSpace[nameSpaceID][nameSpaceID] = 0; + m_LocalToGlobalNameSpace[nameSpaceID][0] = nameSpaceID; +} + +bool PersistentManager::IsFileEmpty (const string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + SerializedFile* serialize = GetSerializedFileInternal (InsertPathNameInternal (pathName, true)); + if (serialize == NULL) + return true; + else + return serialize->IsEmpty (); +} + +#if UNITY_EDITOR +bool PersistentManager::DeleteFile (const string& pathName, DeletionFlags flag) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, true); + if (globalNameSpace == -1) + return false; + + StreamNameSpace& stream = GetStreamNameSpaceInternal (globalNameSpace); + + set<SInt32> objectsInFile; + GetInstanceIDsAtPath (pathName, &objectsInFile); + + for (set<SInt32>::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + MakeObjectUnpersistent (*i, kDontDestroyFromFile); + + if (flag & kDeleteLoadedObjects) + { + for (set<SInt32>::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + { + Object* obj = Object::IDToPointer(*i); + UnloadObject(obj); + } + + #if DEBUGMODE + for (set<SInt32>::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + AssertIf (PPtr<Object> (*i)); + #endif + } + + #if DEBUGMODE + for (set<SInt32>::iterator i=objectsInFile.begin ();i != objectsInFile.end ();++i) + { + if (m_Remapper->GetSerializedFileIndex (*i) != -1) + { + Object* obj = PPtr<Object>(*i); + ErrorStringObject("NOT CORRECTLY UNLOADED!!!!", obj); + } + } + #endif + + if (stream.stream) + CleanupStream(stream); + + m_GlobalToLocalNameSpace[globalNameSpace].clear (); + m_LocalToGlobalNameSpace[globalNameSpace].clear (); + + string absolutePath = RemapToAbsolutePath (pathName); + if (IsFileCreated (absolutePath)) + { + if (!::DeleteFile (absolutePath)) + return false; + } + return true; +} + +#endif + +void PersistentManager::UnloadNonDirtyStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + // printf_console("------ Unloading non dirty stream\n"); + int dirtyStreams = 0; + int loadedStreams = 0; + int unloadedStreams = 0; + for (int i=0;i<m_Streams.size ();i++) + { + StreamNameSpace& nameSpace = m_Streams[i]; + AssertIf (nameSpace.stream == NULL && !m_GlobalToLocalNameSpace[i].empty ()); + if (nameSpace.stream == NULL) + continue; + + if (nameSpace.stream->IsMemoryStream () || nameSpace.stream->IsCachedFileStream()) + { + loadedStreams++; + continue; + } + + bool unloadStream = true; + #if UNITY_EDITOR + unloadStream = !nameSpace.stream->IsFileDirty(); + #endif + + if (unloadStream) + { + unloadedStreams++; + CleanupStream(nameSpace); + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } + else + { + FileIdentifier identifier = PathIDToFileIdentifierInternal(i); + printf_console("Can't unload serialized file because it is dirty: %s\n", identifier.pathName.c_str()); + dirtyStreams++; + loadedStreams++; + } + } + + printf_console("Unloading %d Unused Serialized files (Serialized files now loaded: %d / Dirty serialized files: %d)\n", unloadedStreams, loadedStreams, dirtyStreams); +// printf_console("Streams that can't be unloaded Files %d\n", loadedStreams); +// printf_console("ID mapping count %d -- %d \n", m_Remapper->m_FileToHeapID.size(), m_Remapper->m_FileToHeapID.size() * 24); +// printf_console("RESURRRECT count %d -- %d \n", m_Remapper->m_ResurrectHeapID.size(), m_Remapper->m_ResurrectHeapID.size() * 24); +} + + +void PersistentManager::UnloadStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + for (int i=0;i<m_Streams.size ();i++) + { + StreamNameSpace& nameSpace = m_Streams[i]; + AssertIf (nameSpace.stream == NULL && !m_GlobalToLocalNameSpace[i].empty ()); + if (nameSpace.stream == NULL) + continue; +/* + #if DEBUGMODE + set<SInt32> debug; + m_Remapper->GetAllLoadedObjectsAtPath (i, &debug); + AssertIf (!debug.empty ()); + #endif +*/ + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } +} + +void PersistentManager::UnloadMemoryStreams () +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + for (int i=0;i<m_Streams.size ();i++) + { + StreamNameSpace& nameSpace = m_Streams[i]; + AssertIf (nameSpace.stream == NULL && !m_GlobalToLocalNameSpace[i].empty ()); + if (nameSpace.stream == NULL ) + continue; + + if (!nameSpace.stream->IsMemoryStream () && !nameSpace.stream->IsCachedFileStream()) + continue; + + #if DEBUGMODE + set<SInt32> debug; + m_Remapper->GetAllLoadedObjectsAtPath (i, &debug); + AssertIf (!debug.empty ()); + #endif + + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[i].clear (); + m_LocalToGlobalNameSpace[i].clear (); + } + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.clear(); +} + +void PersistentManager::UnloadStream (const std::string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int nameSpaceID = InsertPathNameInternal (pathName, false); + if (nameSpaceID == -1) + return; + + StreamNameSpace& nameSpace = m_Streams[nameSpaceID]; + if (nameSpace.stream == NULL) + return; + + CleanupStream(nameSpace); + + m_GlobalToLocalNameSpace[nameSpaceID].clear (); + m_LocalToGlobalNameSpace[nameSpaceID].clear (); + + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + m_MemoryLoadedOrCachedPaths.erase(pathName); +} + +bool PersistentManager::HasMemoryOrCachedSerializedFile (const std::string& path) +{ + Mutex::AutoLock lock2 (m_MemoryLoadedOrCachedPathsMutex); + return m_MemoryLoadedOrCachedPaths.count(path) == 1; +} + +void PersistentManager::ResetHighestFileIDAtPath (const string& pathName) +{ + AQUIRE_AUTOLOCK(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, true); + if (globalNameSpace == -1) + return; + + AssertIf (m_Streams[globalNameSpace].stream != NULL); + m_Streams[globalNameSpace].highestID = 0; +} + +#if UNITY_EDITOR +// AwakeFromLoadQueue only supports this in the editor +void PersistentManager::RegisterSafeBinaryReadCallback (SafeBinaryReadCallbackFunction* callback) +{ + AssertIf (gSafeBinaryReadCallback); + gSafeBinaryReadCallback = callback; +} +#endif + +void PersistentManager::RegisterInOrderDeleteCallback (InOrderDeleteCallbackFunction* callback) +{ + AssertIf (gInOrderDeleteCallback); + gInOrderDeleteCallback = callback; +} + +SerializedFile* PersistentManager::GetSerializedFileInternal (const string& path) +{ + return GetSerializedFileInternal(InsertPathNameInternal (path, true)); +} + +SerializedFile* PersistentManager::GetSerializedFileInternal (int serializedFileIndex) +{ + if (serializedFileIndex == -1) + return NULL; + + StreamNameSpace& stream = GetStreamNameSpaceInternal (serializedFileIndex); + return stream.stream; +} + + +bool PersistentManager::IsStreamLoaded (const std::string& pathName) +{ + AQUIRE_AUTOLOCK_WARN_MAIN_THREAD(m_Mutex, gLoadLockPersistentManager); + + int globalNameSpace = InsertPathNameInternal (pathName, false); + if (globalNameSpace == -1) + return false; + + StreamNameSpace& nameSpace = m_Streams[globalNameSpace]; + return nameSpace.stream != NULL; +} + +void PersistentManager::AddStream () +{ + m_Streams.push_back (StreamNameSpace ()); + m_GlobalToLocalNameSpace.push_back (IDRemap ()); + m_LocalToGlobalNameSpace.push_back (IDRemap ()); +} + +void PersistentManager::DestroyFromFileInternal (int memoryID) +{ + SerializedObjectIdentifier identifier; + m_Remapper->InstanceIDToSerializedObjectIdentifier(memoryID, identifier); + SerializedFile* serialize = GetSerializedFileInternal (identifier.serializedFileIndex); + if (serialize) + serialize->DestroyObject (identifier.localIdentifierInFile); +} + +string PersistentManager::RemapToAbsolutePath (const string& path) +{ + UserPathRemap::iterator found = m_UserPathRemap.find(path); + if (found != m_UserPathRemap.end()) + return found->second; + + return PathToAbsolutePath(path); +} + +void PersistentManager::SetPathRemap (const string& path, const string& absoluteRemappedPath) +{ + if (!absoluteRemappedPath.empty()) + { + Assert (m_UserPathRemap.count(path) == 0); + m_UserPathRemap.insert(make_pair(path, absoluteRemappedPath)); + } + else + { + Assert (m_UserPathRemap.count(path) == 1); + m_UserPathRemap.erase(path); + } +} + +#if UNITY_EDITOR +bool PersistentManager::IsClassNonTextSerialized( int cid ) +{ + return m_NonTextSerializedClasses.find (cid) != m_NonTextSerializedClasses.end(); +} +#endif diff --git a/Runtime/Serialize/PersistentManager.h b/Runtime/Serialize/PersistentManager.h new file mode 100644 index 0000000..73f3f67 --- /dev/null +++ b/Runtime/Serialize/PersistentManager.h @@ -0,0 +1,473 @@ +#ifndef PERSISTENTMANAGER_H +#define PERSISTENTMANAGER_H + + +#define SUPPORT_INSTANCE_ID_REMAP_ON_LOAD (UNITY_EDITOR || WEBPLUG) + +class SerializedFile; +class Object; +class TypeTree; +struct FileIdentifier; + +#include <map> +#include <string> +#include <vector> +#include <stack> +#include <deque> +#include <set> +#include "Runtime/Utilities/vector_map.h" +#include "Configuration/UnityConfigure.h" +#include "Runtime/Utilities/CStringHash.h" +#include "Runtime/Threads/Thread.h" +#include "Runtime/Threads/Mutex.h" +#include "WriteData.h" +#include "Runtime/Utilities/MemoryPool.h" +#include "LoadProgress.h" + +using std::map; +using std::set; +using std::vector; +using std::string; +using std::stack; + +class AwakeFromLoadQueue; +class Remapper; + +struct StreamNameSpace +{ + SerializedFile* stream; + LocalIdentifierInFileType highestID; + + StreamNameSpace () { stream = NULL; highestID = 0; } +}; + +enum +{ + kNoError = 0, + kFileCouldNotBeRead = 1, + kTypeTreeIsDifferent = 2, + kFileCouldNotBeWritten = 3 +}; + +enum UnpersistMode { + kDontDestroyFromFile = 0, + kDestroyFromFile = 1 +}; + + +struct ThreadedAwakeData +{ + SInt32 instanceID; + TypeTree* oldType; + Object* object; + bool checkConsistency; /// refactor to safeLoaded + + // Has the object been fully loaded with AwakeFromLoadThreaded. + // We have to make sure the Object* is available already, so that recursive PPtr's to each other from Mono can correctly be resolved. + // In this case, neither object has been fully created, but we can setup pointers between them already. + bool completedThreadAwake; +}; + +struct SerializedObjectIdentifier +{ + SInt32 serializedFileIndex; + LocalIdentifierInFileType localIdentifierInFile; + + SerializedObjectIdentifier (SInt32 inSerializedFileIndex, LocalIdentifierInFileType inLocalIdentifierInFile) + : serializedFileIndex (inSerializedFileIndex) + , localIdentifierInFile (inLocalIdentifierInFile) + { } + + SerializedObjectIdentifier () + : serializedFileIndex (0) + , localIdentifierInFile (0) + { } + + + friend bool operator < (const SerializedObjectIdentifier& lhs, const SerializedObjectIdentifier& rhs) + { + if (lhs.serializedFileIndex < rhs.serializedFileIndex) + return true; + else if (lhs.serializedFileIndex > rhs.serializedFileIndex) + return false; + else + return lhs.localIdentifierInFile < rhs.localIdentifierInFile; + } + + friend bool operator != (const SerializedObjectIdentifier& lhs, const SerializedObjectIdentifier& rhs) + { + return lhs.serializedFileIndex != rhs.serializedFileIndex || lhs.localIdentifierInFile != rhs.localIdentifierInFile; + } +}; + +struct LocalSerializedObjectIdentifier; + +// There are three types of ids. +// fileID, is an id that is local to a file. It ranges from [1 ... kNameSpaceSize] +// heapID, is an id that was allocated for an object that was not loaded from disk. [1 ... infinity] +// pathID, is an id to a file. Every file has a unique id. They are not recycled unless you delete the PersistentManager + +class PersistentManager +{ + protected: + enum + { +// Recursive serialization causes the deflated stream to be reset repeatedly - use bigger cache chunks to limit the impact on load times. +#if UNITY_ANDROID + kCacheSize = 1024 * 256 +#elif UNITY_XBOX360 + kCacheSize = 1024 * 32 +#elif UNITY_WINRT || UNITY_BB10 + kCacheSize = 1024 * 64 +#else + kCacheSize = 1024 * 7 +#endif + }; + + typedef std::pair<SInt32, SInt32> IntPair; + typedef vector_map<SInt32, SInt32, std::less<SInt32>, STL_ALLOCATOR(kMemSerialization, IntPair) > IDRemap; + typedef UNITY_VECTOR(kMemSerialization,StreamNameSpace) StreamContainer; + + StreamContainer m_Streams; + UNITY_VECTOR(kMemSerialization,IDRemap) m_GlobalToLocalNameSpace; + UNITY_VECTOR(kMemSerialization,IDRemap) m_LocalToGlobalNameSpace; + Remapper* m_Remapper; + + typedef std::pair<std::string,std::string> StringPair; + typedef vector_map<std::string, std::string, compare_string_insensitive,STL_ALLOCATOR(kMemSerialization,StringPair)> UserPathRemap; + UserPathRemap m_UserPathRemap; + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + typedef vector_map<SerializedObjectIdentifier, SerializedObjectIdentifier, + std::less<SerializedObjectIdentifier>,STL_ALLOCATOR(kMemSerialization,SerializedObjectIdentifier) > InstanceIDRemap; + InstanceIDRemap m_InstanceIDRemap; + #endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + #if UNITY_EDITOR + UNITY_SET(kMemSerialization,int) m_NonTextSerializedClasses; + #endif + + SInt32 m_CacheCount; + + stack<SInt32, std::deque<SInt32, STL_ALLOCATOR(kMemSerialization,SInt32) > > m_ActiveNameSpace; + int m_Options; + + #if DEBUGMODE + bool m_AllowLoadingFromDisk; + bool m_IsLoadingSceneFile; + int m_PreventLoadingFromFile; + #endif + + UNITY_SET(kMemSerialization, std::string) m_MemoryLoadedOrCachedPaths; + + Mutex m_Mutex; + Mutex m_IntegrationMutex; + + Mutex m_MemoryLoadedOrCachedPathsMutex; + + bool m_AllowIntegrateThreadedObjectsWithTimeout; // Mutex protected by m_IntegrationMutex + +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + MemoryPool m_ThreadedAwakeDataPool; + MemoryPool m_ThreadedAwakeDataPoolMap; + typedef std::list<ThreadedAwakeData, memory_pool_explicit<ThreadedAwakeData> > ThreadedObjectActivationQueue; + typedef std::map<SInt32, ThreadedObjectActivationQueue::iterator, std::less<SInt32>, memory_pool_explicit<ThreadedObjectActivationQueue::iterator> > ThreadedObjectActivationMap; +#else + typedef std::list<ThreadedAwakeData> ThreadedObjectActivationQueue; + typedef std::map<SInt32, ThreadedObjectActivationQueue::iterator> ThreadedObjectActivationMap; +#endif + ThreadedObjectActivationQueue m_ThreadedObjectActivationQueue; // protected by m_IntegrationMutex + ThreadedObjectActivationMap m_ThreadedObjectActivationMap; // protected by m_IntegrationMutex + UNITY_SET(kMemSerialization, SInt32) m_OnDemandThreadLoadedObjects; /// DONT USE POOL HERE NOT THREAD SAFE + + StreamNameSpace& GetStreamNameSpaceInternal (int nameSpaceID); + void DestroyFromFileInternal (int memoryID); + public: + + PersistentManager (int options, int cacheCount); + virtual ~PersistentManager (); + + /// Loads all objects in pathName + /// Returns kNoError, kFileCouldNotBeRead + int LoadFileCompletely (const string& pathname); + + #if UNITY_EDITOR + // Makes an object persistent and generates a unique fileID in pathName + // The object can now be referenced by other objects that write to disk + void MakeObjectPersistent (int heapID, const string& pathName); + // Makes an object persistent if fileID == 0 a new unique fileID in pathName will be generated + // The object can now be referenced by other objects that write to disk + // If the object is already persistent in another file or another fileID it will be destroyed from that file. + void MakeObjectPersistentAtFileID (int heapID, LocalIdentifierInFileType fileID, const string& pathName); + + /// Batch multiple heapID's and fileID's into one path name. + /// on return fileID's will contain the file id's that were generated (if fileIds[i] is non-zero that fileID will be used instead) + enum { kMakePersistentDontRequireToBeLoadedAndDontUnpersist = 1 << 0, kAllowDontSaveObjectsToBePersistent = 1 << 1 }; + void MakeObjectsPersistent (const int* heapIDs, LocalIdentifierInFileType* fileIDs, int size, const string& pathName, int options = 0); + #endif + + // Makes an object unpersistent + void MakeObjectUnpersistent (int memoryID, UnpersistMode unpersistMode); + + bool RemoveObjectsFromPath (const std::string& pathName); + + // Returns the pathname the referenced object is stored at, if its not persistent empty string is returned + string GetPathName (SInt32 memoryID); + // Returns the localFileID the referenced object has inside its file. + + bool InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier); + int SerializedObjectIdentifierToInstanceID (const SerializedObjectIdentifier& identifier); + + LocalIdentifierInFileType GetLocalFileID(SInt32 instanceID); + + // Generates or returns an instanceID from path and fileID which can then + // be used to load the object at that instanceID + ///@TODO: RENAME TO LocalIdentifierInFile + SInt32 GetInstanceIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile); + + // Returns classID from path and fileID. + int GetClassIDFromPathAndFileID (const string& path, LocalIdentifierInFileType localIdentifierInFile); + + // Reads the object referenced by heapID, if there is no object with heapID, the object will first be produced. + // Returns the created and read object, or NULL if the object couldnt be found or was destroyed. + Object* ReadObject (int heapID); + + // Unloads all streams that are open. + // After UnloadStreams is called files may be safely replaced. + // May only be called if there are no dirty files open. + void UnloadStreams (); + void UnloadStream (const std::string& pathName); + + bool IsStreamLoaded (const std::string& pathName); + + #if UNITY_EDITOR + + typedef bool VerifyWriteObjectCallback (Object* verifyDeployment, BuildTargetPlatform target); + + // Writes all persistent objects in memory that are made peristent at pathname to the file + // And completes all write operation (including writing the header) + // Returns the error (kNoError) + // options: kSerializeGameRelease, kSwapEndianess, kBuildPlayerOnlySerializeBuildProperties + int WriteFile (const string& pathName, BuildTargetSelection target = BuildTargetSelection::NoTarget(), int options = 0); + + bool IsClassNonTextSerialized(int cid); + + int WriteFileInternal (const std::string& path, int serializedFileIndex, const WriteData* writeData, int size, VerifyWriteObjectCallback* verifyCallback, BuildTargetSelection target, int options); + + #if UNITY_EDITOR + bool TestNeedWriteFile (const std::string& pathName, const std::set<int>* dirtyPaths = NULL); + bool TestNeedWriteFile (int pathID, const std::set<int>* dirtyPaths = NULL); + #endif + + // Delete file deletes the file referenced by pathName + // Makes all loaded objects unpersistent + // deleteLoadedObjects & kDeleteLoadedObjects -> All objects on the disk will be attempted to be destroyed + // deleteLoadedObjects & kDontDeleteLoadedObjects -> Doesn't delete any loaded objects, but marks them unpersistent from the file. + enum DeletionFlags { kDontDeleteLoadedObjects = 0, kDeleteLoadedObjects = 1 << 0 }; + bool DeleteFile (const string& pathName, DeletionFlags flag); + + void AddNonTextSerializedClass (int classID) { m_NonTextSerializedClasses.insert (classID); } + #endif + + // On return: objects are the instanceIDs of all objects resident in the file referenced by pathName + typedef std::set<SInt32> ObjectIDs; + void GetInstanceIDsAtPath (const string& pathName, ObjectIDs* objects); + void GetInstanceIDsAtPath (const string& pathName, vector<SInt32>* objects); + + void GetLoadedInstanceIDsAtPath (const string& pathName, ObjectIDs* objects); + void GetPersistentInstanceIDsAtPath (const string& pathName, std::set<SInt32>* objects); + + int CountInstanceIDsAtPath (const string& pathName); + + void SetAllowIntegrateThreadedObjectsWithTimeout (bool value); + + bool IsFileEmpty (const string& pathName); + + // Finds out if the referenced object can be loaded. + // By looking for it on the disk. And checking if the classID can be produced. + bool IsObjectAvailable (int heapID); + bool IsObjectAvailableDontCheckActualFile (int heapID); + + void GetAllFileIDs (const string& pathName, vector<LocalIdentifierInFileType>* objects); + + // Finds the + int GetSerializedClassID (int instanceID); + + + + + // Resets the fileIDs. This can only be used if the file has just been deleted. + void ResetHighestFileIDAtPath (const string& pathName); + + // Computes the memoryID (object->GetInstanceID ()) from fileID + // fileID is relative to the file we are currently writing/reading from. + // It can only be called when reading/writing objects in order to + // convert ptrs from file space to global space + void LocalSerializedObjectIdentifierToInstanceIDInternal (const LocalSerializedObjectIdentifier& identifier, SInt32& memoryID); + void LocalSerializedObjectIdentifierToInstanceIDInternal (int activeNameSpace, const LocalSerializedObjectIdentifier& localIdentifier, SInt32& outInstanceID); + + // fileID from memory ID (object->GetInstanceID ()) + // It can only be called when reading/writing objects in order + // to convert ptrs from global space to file space + void InstanceIDToLocalSerializedObjectIdentifierInternal (SInt32 memoryID, LocalSerializedObjectIdentifier& identifier); + + // Translates globalIdentifier.serializedFileIndex from a global index into the local file index based on what file we are currently writing. + // It can only be called when reading/writing objects in order + // to convert ptrs from global space to file space + LocalSerializedObjectIdentifier GlobalToLocalSerializedFileIndexInternal (const SerializedObjectIdentifier& globalIdentifier); + + /// Is this instanceID mapped to the file we are currently writing, + /// in other words is the referenced instanceID read or written from the same file then this will return true. + bool IsInstanceIDFromCurrentFileInternal (SInt32 memoryID); + + #if UNITY_EDITOR + // Hints a fileID to heap id mapping. + // This i used to keep similar instanceID's when entering / exiting playmode + void SuggestFileIDToHeapIDs (const string& pathname, std::map<LocalIdentifierInFileType, SInt32>& fileIDToHeapIDHint); + + int GetSerializedFileIndexFromPath (const std::string& path); + + #endif + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + void RemapInstanceIDOnLoad (const std::string& srcPath, LocalIdentifierInFileType srcLocalFileID, const std::string& dstPath, LocalIdentifierInFileType dstLocalFileID); + void ApplyInstanceIDRemap(SerializedObjectIdentifier& id); + #endif // #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + + /// NOTE: Returns an object that has not been completely initialized (Awake has not been called yet) + Object* ReadObjectThreaded (int heapID); + + /// Check if we have objects to integrate, ie. if calling IntegrateThreadedObjects will perform some useful work + bool HasThreadedObjectsToIntegrate (); + + /// Integrates all thread loaded objects into the world (Called from PlayerLoop) + void IntegrateThreadedObjects (float timeout); + + /// Called from outise the loading thread (non-main thread), allows IntegrateThreadedObjects to be called with time slicing + /// Stalls until all objects have been integrated + void AllowIntegrationWithTimeoutAndWait (); + + void IntegrateAllThreadedObjects (); + + + void PrepareAllThreadedObjectsStep1 (AwakeFromLoadQueue& awakeQueue); + void IntegrateAllThreadedObjectsStep2 (AwakeFromLoadQueue& awakeQueue/*, bool loadScene*/); + + + + /// Load the entire file from a different thread + int LoadFileCompletelyThreaded (const std::string& pathname, LocalIdentifierInFileType* fileIDs, SInt32* instanceIDs, int size, bool loadScene, LoadProgress* loadProgress); + + /// Loads a number of objects threaded + void LoadObjectsThreaded (SInt32* heapIDs, int size, LoadProgress* loadProgress); + + #if SUPPORT_THREADS + const Mutex& GetMutex () { return m_Mutex; } + Thread::ThreadID GetMainThreadID () { return Thread::mainThreadId; } + #endif + + #if DEBUGMODE + /// Allows you to assert on any implicit loading operations for a specific file + void SetDebugAssertLoadingFromFile (const std::string& path); + /// Allows you to assert on any implicit loading operations, because for example when unloading objects that is usually not desired. + void SetDebugAssertAllowLoadingAnything (bool allowLoading) { m_AllowLoadingFromDisk = allowLoading; } + + void SetIsLoadingSceneFile(bool inexplicit) { m_IsLoadingSceneFile = inexplicit; } + #endif + + void Lock(); + void Unlock(); + /// Loads the contents of the object from disk again + /// Performs serialization and calls AwakeFromLoad + bool ReloadFromDisk (Object* obj); + + /// Load a memory stream directly from memory + /// - You should call this function only on assets that have been writting using kSerializeGameReleaswe + bool LoadMemoryBlockStream (const std::string& pathName, UInt8** data, int offset, int end, const char* url = NULL); + + // Loads a file at actualAbsolutePath and pretends that it is actually at path. + /// - You should call this function only on assets that have been writting using kSerializeGameReleaswe + bool LoadCachedFile (const std::string& path, const std::string actualAbsolutePath); + + void UnloadMemoryStreams (); + + // A registered SafeBinaryReadCallbackFunction will be called when an objects old typetree is different from the new one + // and variables might have gone away, added, or changed + typedef void SafeBinaryReadCallbackFunction (Object& object, const TypeTree& oldTypeTree); + static void RegisterSafeBinaryReadCallback (SafeBinaryReadCallbackFunction* callback); + + // In order for DeleteFile to work you have to support a callback that deletes the objects referenced by instanceID + typedef void InOrderDeleteCallbackFunction (const set<SInt32>& objects, bool safeDestruction); + static void RegisterInOrderDeleteCallback (InOrderDeleteCallbackFunction* callback); + + /// Thread locking must be performed from outside using Lock/Unlock + SerializedFile* GetSerializedFileInternal (const string& path); + SerializedFile* GetSerializedFileInternal (int serializedFileIndex); + + void SetPathRemap (const string& path, const string& absoluteRemappedPath); + + // Non-Locking method to find out if a memorystream or cached file is set up. + bool HasMemoryOrCachedSerializedFile (const std::string& path); + + #if SUPPORT_SERIALIZATION_FROM_DISK + bool LoadExternalStream (const std::string& pathName, const std::string& absolutePath, int flags, int readOffset = 0); + #endif + + void UnloadNonDirtyStreams (); + + /// NOTE: Function postfixed Internal are not thread safe and you have to call PersistentManager.Lock / Unlock prior to calling them from outside persistentmanager + + + //// Subclasses have to override these methods which map from PathIDs to FileIdentifier + /// Maps a pathname/fileidentifier to a pathID. If the pathname is not yet known, you have to call AddStream (). + /// The pathIDs start at 0 and increment by 1 + virtual int InsertFileIdentifierInternal (FileIdentifier file, bool create) = 0; + + std::string RemapToAbsolutePath (const std::string& path); + + void DoneLoadingManagers(); + + protected: + + virtual int InsertPathNameInternal (const std::string& pathname, bool create) = 0; + + /// maps a pathID to a pathname/file guid/fileidentifier. + /// (pathID can be assumed to be allocated before with InsertPathName) + virtual string PathIDToPathNameInternal (int pathID) = 0; + virtual FileIdentifier PathIDToFileIdentifierInternal (int pathID) = 0; + + /// Adds a new empty stream. Used by subclasses inside InsertPathName when a new pathID has to be added + void AddStream (); + + private: + + void CleanupStreamAndNameSpaceMapping (unsigned pathID); + + void RegisterAndAwakeThreadedObjectAndUnlockIntegrationMutex (const ThreadedAwakeData& awake); + + ThreadedAwakeData* CreateThreadActivationQueueEntry (SInt32 instanceID); + void SetupThreadActivationQueueObject (ThreadedAwakeData& data, TypeTree* oldType, bool didTypeTreeChange); + + /// Goes through object activation queue and calls AwakeFromLoad if it has been serialized already but not AwakeFromLoad called. + Object* LoadFromActivationQueue (int heapID); + Object* GetFromActivationQueue (int heapID); + bool FindInActivationQueue (int heapID); + void CheckInstanceIDsLoaded (SInt32* heapIDs, int size); + + protected: + + void PostLoadStreamNameSpace (StreamNameSpace& nameSpace, int namespaceID); + +#if UNITY_EDITOR + bool TestNeedWriteFileInternal (int pathID, const std::set<int>* cachedDirtyPathsHint); +#endif + + friend class Object; +}; + +PersistentManager& GetPersistentManager (); +PersistentManager* GetPersistentManagerPtr (); + +void CleanupPersistentManager(); + +#endif diff --git a/Runtime/Serialize/Remapper.h b/Runtime/Serialize/Remapper.h new file mode 100644 index 0000000..6e1debc --- /dev/null +++ b/Runtime/Serialize/Remapper.h @@ -0,0 +1,297 @@ +#ifndef REMAPPER_H +#define REMAPPER_H + +#include <limits> +#include "Runtime/Utilities/MemoryPool.h" +#include "Configuration/UnityConfigure.h" + +class Remapper +{ + public: +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP + MemoryPool m_SerializedObjectIdentifierPool; + typedef std::map<SerializedObjectIdentifier, SInt32, std::less<SerializedObjectIdentifier>, memory_pool_explicit<std::pair<const SerializedObjectIdentifier, SInt32> > > FileToHeapIDMap; + typedef std::map<SInt32, SerializedObjectIdentifier, std::less<SInt32>, memory_pool_explicit<std::pair<const SInt32, SerializedObjectIdentifier> > > HeapIDToFileMap; +#else + typedef std::map<SerializedObjectIdentifier, SInt32, std::less<SerializedObjectIdentifier> > FileToHeapIDMap; + typedef std::map<SInt32, SerializedObjectIdentifier, std::less<SInt32> > HeapIDToFileMap; +#endif + + typedef FileToHeapIDMap::iterator FileToHeapIDIterator; + typedef HeapIDToFileMap::iterator HeapIDToFileIterator; + + FileToHeapIDMap m_FileToHeapID; + HeapIDToFileMap m_HeapIDToFile; + #if UNITY_EDITOR + vector_set<SInt32> m_LoadingSceneInstanceIDs; + #endif + + + + // Instance ID's are simply allocated in an increasing index + int m_HighestMemoryID; + + // When loading scenes we can fast path because objects are not kept persistent / unloaded / loaded again etc. + // So we just preallocate a bunch of id's and use those without going through a lot of table lookups. + int m_ActivePreallocatedIDBase; + int m_ActivePreallocatedIDEnd; + int m_ActivePreallocatedPathID; + + Remapper () +// map node contains 3 pointers (left, right, parent) + +#if ENABLE_CUSTOM_ALLOCATORS_FOR_STDMAP +: m_SerializedObjectIdentifierPool( false, "Remapper pool", sizeof (SerializedObjectIdentifier) + sizeof(SInt32)*2 + sizeof(void*)*3, 16 * 1024) +, m_FileToHeapID (std::less<SerializedObjectIdentifier> (), m_SerializedObjectIdentifierPool) +, m_HeapIDToFile (std::less<SInt32> (), m_SerializedObjectIdentifierPool) +#endif + { m_HighestMemoryID = 0; m_ActivePreallocatedIDBase = 0; m_ActivePreallocatedIDEnd = 0; m_ActivePreallocatedPathID = -1; } + + + void PreallocateIDs (LocalIdentifierInFileType highestFileID, int pathID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf(pathID == -1); + m_HighestMemoryID += 2; + m_ActivePreallocatedIDBase = m_HighestMemoryID; + m_HighestMemoryID += highestFileID * 2; + m_ActivePreallocatedIDEnd = m_HighestMemoryID; + //printf_console("Preallocating %d .. %d\n", m_ActivePreallocatedIDBase, m_ActivePreallocatedIDEnd); + m_ActivePreallocatedPathID = pathID; + } + + void ClearPreallocateIDs () + { + AssertIf(m_ActivePreallocatedPathID == -1); + m_ActivePreallocatedIDBase = 0; + m_ActivePreallocatedIDEnd = 0; + m_ActivePreallocatedPathID = -1; + } + + void Remove (int memoryID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + + HeapIDToFileIterator i = m_HeapIDToFile.find (memoryID); + if (i == m_HeapIDToFile.end ()) + return; + + FileToHeapIDIterator j = m_FileToHeapID.find (i->second); + AssertIf (j == m_FileToHeapID.end ()); + SerializedObjectIdentifier bug = j->first; + + m_HeapIDToFile.erase (i); + m_FileToHeapID.erase (j); + AssertIf (m_FileToHeapID.find (bug) != m_FileToHeapID.end ()); + } + + void RemoveCompletePathID (int serializedFileIndex, vector<SInt32>& objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + + SerializedObjectIdentifier proxy; + proxy.serializedFileIndex = serializedFileIndex; + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::min(); + + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::max(); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + for (FileToHeapIDIterator i=begin;i != end;i++) + { + ErrorIf(i->first.serializedFileIndex != serializedFileIndex); + m_HeapIDToFile.erase (m_HeapIDToFile.find(i->second)); + objects.push_back(i->second); + } + m_FileToHeapID.erase(begin, end); + } + + bool IsSceneID (int memoryID) + { +#if UNITY_EDITOR + return m_LoadingSceneInstanceIDs.count(memoryID) != 0; +#endif + return IsPreallocatedID (memoryID); + } + + bool IsPreallocatedID (int memoryID) + { + return m_ActivePreallocatedPathID != -1 && memoryID >= m_ActivePreallocatedIDBase && memoryID <= m_ActivePreallocatedIDEnd; + } + + bool InstanceIDToSerializedObjectIdentifier (int instanceID, SerializedObjectIdentifier& identifier) + { + if (IsPreallocatedID(instanceID)) + { + identifier.serializedFileIndex = m_ActivePreallocatedPathID; + identifier.localIdentifierInFile = (instanceID - m_ActivePreallocatedIDBase) / 2; + return true; + } + + HeapIDToFileIterator i = m_HeapIDToFile.find (instanceID); + if (i == m_HeapIDToFile.end ()) + { + identifier.serializedFileIndex = -1; + identifier.localIdentifierInFile = 0; + return false; + } + identifier = i->second; + + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + + return true; + } + + + int GetSerializedFileIndex (int memoryID) + { + SerializedObjectIdentifier identifier; + InstanceIDToSerializedObjectIdentifier (memoryID, identifier); + return identifier.serializedFileIndex; + } + + bool IsSetup (const SerializedObjectIdentifier& identifier) + { + return m_FileToHeapID.find (identifier) != m_FileToHeapID.end (); + } + + bool IsHeapIDSetup (int memoryID) + { + AssertIf(m_ActivePreallocatedPathID != -1); + return m_HeapIDToFile.count(memoryID); + } + + int GetOrGenerateMemoryID (const SerializedObjectIdentifier& identifier) + { + if (identifier.serializedFileIndex == -1) + return 0; + + if (m_ActivePreallocatedPathID != -1 && m_ActivePreallocatedPathID == identifier.serializedFileIndex) + { + return identifier.localIdentifierInFile * 2 + m_ActivePreallocatedIDBase; + } + + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + std::pair<FileToHeapIDIterator, bool> inserted = m_FileToHeapID.insert (std::make_pair (identifier, 0)); + if (inserted.second) + { + int memoryID = 0; + + m_HighestMemoryID += 2; + memoryID = m_HighestMemoryID; + + inserted.first->second = memoryID; + + AssertIf (m_HeapIDToFile.find (memoryID) != m_HeapIDToFile.end ()); + m_HeapIDToFile.insert (std::make_pair (memoryID, identifier)); + + return memoryID; + } + else + return inserted.first->second; + } + + void SetupRemapping (int memoryID, const SerializedObjectIdentifier& identifier) + { + AssertIf(m_ActivePreallocatedPathID != -1); + #if LOCAL_IDENTIFIER_IN_FILE_SIZE != 32 + -- fix this, should we use UInt32 for localIdentifierInFile? + USInt64 debugLocalIdentifier = identifier.localIdentifierInFile; + AssertIf(debugLocalIdentifier >= (1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE) || debugLocalIdentifier <= -(1ULL << LOCAL_IDENTIFIER_IN_FILE_SIZE)); + #endif + + if (m_HeapIDToFile.find (memoryID) != m_HeapIDToFile.end ()) + { + m_FileToHeapID.erase(m_HeapIDToFile.find (memoryID)->second); + m_HeapIDToFile.erase(memoryID); + } + + if (m_FileToHeapID.find (identifier) != m_FileToHeapID.end ()) + { + m_HeapIDToFile.erase(m_FileToHeapID.find (identifier)->second); + m_FileToHeapID.erase(identifier); + } + + m_HeapIDToFile[memoryID] = identifier; + m_FileToHeapID[identifier] = memoryID; + +/* +// This code asserts more when something goes wrong but also in edge cases that are allowed. + SerializedObjectIdentifier id; + id.fileID = fileID; + id.pathID = pathID; + + HeapIDToFileIterator inserted; + inserted = m_HeapIDToFile.insert (std::make_pair (memoryID, id)).first; + AssertIf (inserted->second != id); + inserted->second = id; + + FileToHeapIDIterator inserted2; + #if DEBUGMODE + inserted2 = m_FileToHeapID.find (id); + AssertIf (inserted2 != m_FileToHeapID.end () && inserted2->second != memoryID); + #endif + + inserted2 = m_FileToHeapID.insert (std::make_pair (id, memoryID)).first; + AssertIf (inserted2->second != memoryID); + inserted2->second = memoryID; +*/ + } + + void GetAllLoadedObjectsAtPath (int pathID, set<SInt32>* objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf (objects == NULL); + + SerializedObjectIdentifier proxy; + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::min (); + proxy.serializedFileIndex = pathID; + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::max (); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + + for (FileToHeapIDIterator i=begin;i != end;++i) + { + int instanceID = i->second; + Object* o = Object::IDToPointer (instanceID); + if (o) + objects->insert (instanceID); + } + } + + void GetAllPersistentObjectsAtPath (int pathID, set<SInt32>* objects) + { + AssertIf(m_ActivePreallocatedPathID != -1); + AssertIf (objects == NULL); + + SerializedObjectIdentifier proxy; + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::min (); + proxy.serializedFileIndex = pathID; + FileToHeapIDIterator begin = m_FileToHeapID.lower_bound (proxy); + proxy.localIdentifierInFile = std::numeric_limits<LocalIdentifierInFileType>::max (); + FileToHeapIDIterator end = m_FileToHeapID.upper_bound (proxy); + + for (FileToHeapIDIterator i=begin;i != end;++i) + { + int instanceID = i->second; + objects->insert(instanceID); + } + } + + int GetHighestInUseHeapID () + { + if (!m_HeapIDToFile.empty()) + return m_HeapIDToFile.rbegin ()->first; + else + return 0; + } +}; + +#endif diff --git a/Runtime/Serialize/SerializationMetaFlags.h b/Runtime/Serialize/SerializationMetaFlags.h new file mode 100644 index 0000000..fc47207 --- /dev/null +++ b/Runtime/Serialize/SerializationMetaFlags.h @@ -0,0 +1,292 @@ +#ifndef SERIALIZATIONMETAFLAGS_H +#define SERIALIZATIONMETAFLAGS_H + +#include "Runtime/Utilities/EnumFlags.h" + +/// Meta flags can be used like this: +/// transfer.Transfer (someVar, "varname", kHideInEditorMask); +/// The proxytransfer for example reads the metaflag mask and stores it in the TypeTree +enum TransferMetaFlags +{ + kNoTransferFlags = 0, + /// Putting this mask in a transfer will make the variable be hidden in the property editor + kHideInEditorMask = 1 << 0, + + /// Makes a variable not editable in the property editor + kNotEditableMask = 1 << 4, + + /// There are 3 types of PPtrs: kStrongPPtrMask, default (weak pointer) + /// a Strong PPtr forces the referenced object to be cloned. + /// A Weak PPtr doesnt clone the referenced object, but if the referenced object is being cloned anyway (eg. If another (strong) pptr references this object) + /// this PPtr will be remapped to the cloned object + /// If an object referenced by a WeakPPtr is not cloned, it will stay the same when duplicating and cloning, but be NULLed when templating + kStrongPPtrMask = 1 << 6, + // unused = 1 << 7, + + /// kEditorDisplaysCheckBoxMask makes an integer variable appear as a checkbox in the editor + kEditorDisplaysCheckBoxMask = 1 << 8, + + // unused = 1 << 9, + // unused = 1 << 10, + + /// Show in simplified editor + kSimpleEditorMask = 1 << 11, + + /// When the options of a serializer tells you to serialize debug properties kSerializeDebugProperties + /// All debug properties have to be marked kDebugPropertyMask + /// Debug properties are shown in expert mode in the inspector but are not serialized normally + kDebugPropertyMask = 1 << 12, + + kAlignBytesFlag = 1 << 14, + kAnyChildUsesAlignBytesFlag = 1 << 15, + kIgnoreWithInspectorUndoMask= 1 << 16, + + // unused = 1 << 18, + + // Ignore this property when reading or writing .meta files + kIgnoreInMetaFiles = 1 << 19, + + // When reading meta files and this property is not present, read array entry name instead (for backwards compatibility). + kTransferAsArrayEntryNameInMetaFiles = 1 << 20, + + // When writing YAML Files, uses the flow mapping style (all properties in one line, with "{}"). + kTransferUsingFlowMappingStyle = 1 << 21, + + // Tells SerializedProperty to generate bitwise difference information for this field. + kGenerateBitwiseDifferences = 1 << 22, + + kDontAnimate = 1 << 23, + +}; +ENUM_FLAGS(TransferMetaFlags); + +enum TransferInstructionFlags +{ + kNoTransferInstructionFlags = 0, + + kNeedsInstanceIDRemapping = 1 << 0, // Should we convert PPtrs into pathID, fileID using the PerisistentManager or should we just store the memory InstanceID in the fileID? + kAssetMetaDataOnly = 1 << 1, // Only serialize data needed for .meta files + kYamlGlobalPPtrReference = 1 << 2, + #if UNITY_EDITOR + kLoadAndUnloadAssetsDuringBuild = 1 << 3, + kSerializeDebugProperties = 1 << 4, // Should we serialize debug properties (eg. Serialize mono private variables) + #endif + kIgnoreDebugPropertiesForIndex = 1 << 5, // Should we ignore Debug properties when calculating the TypeTree index + #if UNITY_EDITOR + kBuildPlayerOnlySerializeBuildProperties = 1 << 6, // Used by eg. build player to make materials cull any properties are aren't used anymore ! + #endif + kWorkaround35MeshSerializationFuckup = 1 << 7, + + kSerializeGameRelease = 1 << 8, // Should Transfer classes use optimized reading. Allowing them to read memory directly that normally has a type using ReadDirect. + kSwapEndianess = 1 << 9, // Should we swap endianess when reading / writing a file + kSaveGlobalManagers = 1 << 10, // Should global managers be saved when writing the game build + kDontReadObjectsFromDiskBeforeWriting = 1 << 11, + kSerializeMonoReload = 1 << 12, // Should we backupmono mono variables for an assembly reload? + kDontRequireAllMetaFlags = 1 << 13, // Can we fast path calculating all meta data. This lets us skip a bunch of code when serializing mono data. + kSerializeForPrefabSystem = 1 << 14, + #if UNITY_EDITOR + kWarnAboutLeakedObjects = 1 << 15, + // Unused = 1 << 16, + // Unused = 1 << 17, + kEditorPlayMode = 1 << 18, + kBuildResourceImage = 1 << 19, + kSerializeEditorMinimalScene = 1 << 21, + kGenerateBakedPhysixMeshes = 1 << 22, + #endif + kThreadedSerialization = 1 << 23, + kIsBuiltinResourcesFile = 1 << 24, + kPerformUnloadDependencyTracking = 1 << 25, + kDisableWriteTypeTree = 1 << 26, + kAutoreplaceEditorWindow = 1 << 27,// Editor only + kSerializeForInspector = 1 << 29, + kSerializedAssetBundleVersion = 1 << 30, // When writing (typetrees disabled), allow later Unity versions an attempt to read SerializedFile. + kAllowTextSerialization = 1 << 31 +}; +ENUM_FLAGS(TransferInstructionFlags); + +enum BuildAssetBundleOptions +{ + kAssetBundleUncompressed = 1 << 11, + kAssetBundleCollectDependencies = 1 << 20, + kAssetBundleIncludeCompleteAssets = 1 << 21, + kAssetBundleDisableWriteTypeTree = 1 << 26, + kAssetBundleDeterministic = 1 << 28, +}; +ENUM_FLAGS(BuildAssetBundleOptions); + + +enum ActiveResourceImage +{ + kResourceImageNotSupported = -2, + kResourceImageInactive = -1, + kGPUResourceImage = 0, + kResourceImage = 1, + kStreamingResourceImage = 2, + kNbResourceImages = 3 +}; + +/// This needs to be in Sync with BuildTarget in C# +enum BuildTargetPlatform +{ + kBuildNoTargetPlatform = -2, + kBuildAnyPlayerData = -1, + kBuildValidPlayer = 1, + + // We don't support building for these any more, but we still need the constants for asset bundle + // backwards compatibility. + kBuildStandaloneOSXPPC = 3, + + kBuildStandaloneOSXIntel = 4, + kBuildStandaloneOSXIntel64 = 27, + kBuildStandaloneOSXUniversal = 2, + kBuildStandaloneWinPlayer = 5, + kBuildWebPlayerLZMA = 6, + kBuildWebPlayerLZMAStreamed = 7, + kBuildWii = 8, + kBuild_iPhone = 9, + kBuildPS3 = 10, + kBuildXBOX360 = 11, + // was kBuild_Broadcom = 12, + kBuild_Android = 13, + kBuildWinGLESEmu = 14, + // was kBuildWinGLES20Emu = 15, + kBuildNaCl = 16, + kBuildStandaloneLinux = 17, + kBuildFlash = 18, + kBuildStandaloneWin64Player = 19, + kBuildWebGL = 20, + kBuildMetroPlayerX86 = 21, + kBuildMetroPlayerX64 = 22, + kBuildMetroPlayerARM = 23, + kBuildStandaloneLinux64 = 24, + kBuildStandaloneLinuxUniversal = 25, + kBuildWP8Player = 26, + kBuildBB10 = 28, + kBuildTizen = 29, + kBuildPlayerTypeCount = 30, +}; + +struct BuildUsageTag +{ + bool forceTextureReadable; + bool strippedPrefabObject; + UInt32 meshUsageFlags; + UInt32 meshSupportedChannels; + + BuildUsageTag () + { + forceTextureReadable = false; + meshUsageFlags = 0; + meshSupportedChannels = 0; + strippedPrefabObject = false; + } +}; + + +struct BuildTargetSelection +{ + BuildTargetPlatform platform; + int subTarget; + + BuildTargetSelection() : platform(kBuildNoTargetPlatform), subTarget(0) { } + BuildTargetSelection(BuildTargetPlatform platform_, int subTarget_) : platform(platform_), subTarget(subTarget_) {} + + bool operator == (const BuildTargetSelection& rhs) const + { + if (platform != rhs.platform) + return false; + if (subTarget != rhs.subTarget) + return false; + + return true; + } + bool operator != (const BuildTargetSelection& rhs) const + { + return ! operator == (rhs); + } + + static BuildTargetSelection NoTarget() { return BuildTargetSelection(kBuildNoTargetPlatform,0); } +}; + + +enum WebPlayerBuildSubTarget +{ + kWebBuildSubtargetDefault = 0, + kWebBuildSubtargetDirect3D = 1, // windows only (D3D9 & D3D11) + kWebBuildSubtargetOpenGL = 2, // non-windows only (OpenGL) +}; + + +/// This needs to be in Sync with XboxRunMethod in C# +enum XboxBuildSubtarget +{ + kXboxBuildSubtargetDevelopment = 0, + kXboxBuildSubtargetMaster = 1, + kXboxBuildSubtargetDebug = 2 +}; + +/// This needs to be in Sync with WiiBuildDebugLevel in C# +enum WiiBuildDebugLevel +{ + kWiiBuildDebugLevel_Full = 0, + kWiiBuildDebugLevel_Minimal = 1, + kWiiBuildDebugLevel_None = 2, +}; + +/// This needs to be in Sync with XboxRunMethod in C# +enum XboxRunMethod +{ + kXboxRunMethodHDD = 0, + kXboxRunMethodDiscEmuFast = 1, + kXboxRunMethodDiscEmuAccurate = 2 +}; + +/// This needs to be in Sync with AndroidBuildSubtarget in C# +enum AndroidBuildSubtarget +{ + kAndroidBuildSubtarget_Generic = 0, + kAndroidBuildSubtarget_DXT = 1, + kAndroidBuildSubtarget_PVRTC = 2, + kAndroidBuildSubtarget_ATC = 3, + kAndroidBuildSubtarget_ETC = 4, + kAndroidBuildSubtarget_ETC2 = 5, + kAndroidBuildSubtarget_ASTC = 6, +}; + +/// This needs to be in Sync with BB10BuildSubtarget in C# +enum BlackBerryBuildSubtarget +{ + kBlackBerryBuildSubtarget_Generic = 0, + kBlackBerryBuildSubtarget_PVRTC = 1, + kBlackBerryBuildSubtarget_ATC = 2, + kBlackBerryBuildSubtarget_ETC = 3 +}; + +/// This needs to be in Sync with BB10BuildType in C# +enum BlackBerryBuildType +{ + kBlackBerryBuildType_Debug = 0, + kBlackBerryBuildType_Submission = 1 +}; + +/// This needs to be in Sync with BuildOptions in C# +enum BuildPlayerOptions +{ + kBuildPlayerOptionsNone = 0, + kDevelopmentBuild = 1 << 0, + kAutoRun = 1 << 2, + kSelectBuiltPlayer = 1 << 3, + kBuildAdditionalStreamedScenes = 1 << 4, + kAcceptExternalModificationsToPlayer = 1 << 5, + kInstallInBuildsFolder = 1 << 6, + kWebPlayerOfflineDeployment = 1 << 7, + kConnectWithProfiler = 1 << 8, + kAllowDebugging = 1 << 9, + kSymlinkLibraries = 1 << 10, + kBuildPlayerUncompressed = 1 << 11, + kConnectToHost = 1 << 12, + kDeployOnline = 1 << 13, + kHeadlessModeEnabled = 1 << 14 +}; + +#endif diff --git a/Runtime/Serialize/SerializationTests.cpp b/Runtime/Serialize/SerializationTests.cpp new file mode 100644 index 0000000..def94dd --- /dev/null +++ b/Runtime/Serialize/SerializationTests.cpp @@ -0,0 +1,242 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" + +#if ENABLE_UNIT_TESTS + +#include "Runtime/Testing/Testing.h" +#include "Runtime/Testing/TestFixtures.h" + +SUITE (SerializationTests) +{ + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidReadExistingProperty) + { + float m_FloatProperty; + TRANSFER (m_FloatProperty); + if (transfer.IsReading ()) + { + CHECK (transfer.DidReadLastProperty ()); + } + } + + TEST_FIXTURE (DidReadExistingPropertyTestFixture, SafeBinaryRead_DidReadLastProperty_WithExistingProperty_IsTrue) + { + DoSafeBinaryTransfer (); + } + + TEST_FIXTURE (DidReadExistingPropertyTestFixture, YAMLRead_DidReadLastProperty_WithExistingProperty_IsTrue) + { + DoTextTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidNotReadMissingProperty) + { + float m_Foobar; + TRANSFER (m_Foobar); + + if (transfer.IsReading ()) + { + UnityStr value = "foobar"; + TRANSFER (value); + + CHECK (!transfer.DidReadLastProperty ()); + CHECK (value == "foobar"); + } + } + + TEST_FIXTURE (DidNotReadMissingPropertyTestFixture, SafeBinaryRead_DidReadLastProperty_WithMissingProperty_IsFalse) + { + DoSafeBinaryTransfer (); + } + + TEST_FIXTURE (DidNotReadMissingPropertyTestFixture, YAMLRead_DidReadLastProperty_WithMissingProperty_IsFalse) + { + DoTextTransfer (); + } + + //------------------------------------------------------------------------- + +#define kDoubleValue 0.1 +#define kFloatValue -2.5f +#define kIntValue 1337 +#define kLongLongValue 1234567890123456789LL +#define kCharValue 'X' +#define kBoolValue true +#define kStringValue "UnityFTW" +#define kVectorSize 3 + +template<class T> +struct FloatingPointConsistencyTest +{ + #define kNumFloatValues 12 + T values[kNumFloatValues]; + + T Get(int i) + { + switch (i) + { + case 0: return std::numeric_limits<T>::min(); + case 1: return std::numeric_limits<T>::max(); + case 2: return std::numeric_limits<T>::denorm_min(); + case 3: return std::numeric_limits<T>::infinity(); + case 4: return -std::numeric_limits<T>::infinity(); + case 5: return std::numeric_limits<T>::quiet_NaN(); + case 6: return std::numeric_limits<T>::epsilon(); + case 7: return -0.0; + case 8: return (T)12345678901234567890.123456789012345678900; + case 9: return (T)0.1; + case 10: return (T)(1.0 / 3.0); + case 11: return (T)(3 * 1024 * 1024 * 0.19358); + default: ErrorString("Should not happen!"); return 0; + } + } + + void FillStruct () + { + for (int i=0; i<kNumFloatValues; i++) + values[i] = Get(i); + } + + void VerifyStruct () + { + for (int i=0; i<kNumFloatValues; i++) + { + T expected = Get(i); + + // Use memcmp instead of == to test for negative zero and NaN. + CHECK (memcmp (&expected, values+i, sizeof(T)) == 0); + } + } + + DECLARE_SERIALIZE (FloatingPointConsistencyTest) +}; + +template<class T> +template<class TransferFunction> inline +void FloatingPointConsistencyTest<T>::Transfer (TransferFunction& transfer) +{ + for (int i=0; i<kNumFloatValues; i++) + TRANSFER(values[i]); +} + +struct TestStruct { + float m_Float; + int m_Int; + long long m_LongLong; + char m_Char; + bool m_Bool; + + void FillStruct () + { + m_Float = kFloatValue; + m_Int = kIntValue; + m_Char = kCharValue; + m_Bool = kBoolValue; + m_LongLong = kLongLongValue; + } + + void VerifyStruct () + { + CHECK_EQUAL (m_Float, kFloatValue); + CHECK_EQUAL (m_Int, kIntValue); + CHECK_EQUAL (m_Char, kCharValue); + CHECK_EQUAL (m_Bool, kBoolValue); + CHECK_EQUAL (m_LongLong, kLongLongValue); + } + DECLARE_SERIALIZE (TestStruct) +}; + +struct TestStruct2 { + FloatingPointConsistencyTest<float> m_FloatTest; + FloatingPointConsistencyTest<double> m_DoubleTest; + UnityStr m_String; + TestStruct m_Struct; + std::vector<TestStruct> m_Vector; + std::map<int, TestStruct> m_Map; + char m_TypelessData[kVectorSize]; + + void FillStruct () + { + m_FloatTest.FillStruct(); + m_DoubleTest.FillStruct(); + m_String = kStringValue; + m_Struct.FillStruct(); + m_Vector.resize(kVectorSize); + for (int i=0;i<kVectorSize;i++) + m_Vector[i].FillStruct(); + m_Map[42].FillStruct(); + m_Map[666].FillStruct(); + m_Map[23].FillStruct(); + for (int i=0;i<kVectorSize;i++) + m_TypelessData[i] = i; + } + + void VerifyStruct () + { + CHECK_EQUAL (m_String, kStringValue); + m_Struct.VerifyStruct(); + CHECK_EQUAL (m_Vector.size(), kVectorSize); + for (int i=0;i<kVectorSize;i++) + m_Vector[i].VerifyStruct(); + + m_Map[42].VerifyStruct(); + m_Map[666].VerifyStruct(); + m_Map[23].VerifyStruct(); + for (int i=0;i<kVectorSize;i++) + CHECK_EQUAL (m_TypelessData[i], i); + m_FloatTest.VerifyStruct(); + m_DoubleTest.VerifyStruct(); + } + + DECLARE_SERIALIZE (TestStruct2) +}; + +template<class TransferFunction> inline +void TestStruct::Transfer (TransferFunction& transfer) +{ + TRANSFER(m_Float); + TRANSFER(m_Int); + TRANSFER(m_Char); + TRANSFER(m_Bool); + TRANSFER(m_LongLong); +} + +template<class TransferFunction> inline +void TestStruct2::Transfer (TransferFunction& transfer) +{ + TRANSFER(m_FloatTest); + TRANSFER(m_DoubleTest); + TRANSFER(m_String); + TRANSFER(m_Struct); + TRANSFER(m_Vector); + + TRANSFER(m_Map); + transfer.TransferTypelessData (kVectorSize, m_TypelessData, 0); +} + +#if SUPPORT_TEXT_SERIALIZATION +TEST (SerialializeYAMLStruct) +{ + TestStruct2 input; + input.FillStruct (); + + YAMLWrite write (0); + input.Transfer( write ); + std::string str; + write.OutputToString(str); + TestStruct2 output; + + YAMLRead read (str.c_str(), str.size(), 0); + output.Transfer( read ); + + output.VerifyStruct(); + +} //TEST +#endif + +} //SUITE + +#endif diff --git a/Runtime/Serialize/SerializeConversion.h b/Runtime/Serialize/SerializeConversion.h new file mode 100644 index 0000000..782ea5b --- /dev/null +++ b/Runtime/Serialize/SerializeConversion.h @@ -0,0 +1,34 @@ +#ifndef SERIALIZECONVERSION_H +#define SERIALIZECONVERSION_H + +#if SUPPORT_SERIALIZED_TYPETREES +#include "Runtime/Serialize/TransferFunctions/SerializeTransfer.h" +#include "SerializeTraits.h" + +// Trys to convert from an old type to a new one +template<class OldFormat, class NewFormat> +bool StdTemplateConversionFunction (void* inData, SafeBinaryRead& transfer) +{ + NewFormat& data = *reinterpret_cast<NewFormat*> (inData); + const TypeTree& oldTypeTree = transfer.GetActiveOldTypeTree (); + AssertIf (SerializeTraits<OldFormat>::GetTypeString (NULL) != oldTypeTree.m_Type); + OldFormat oldData; + + SafeBinaryRead safeRead; + CachedReader& temp = safeRead.Init (transfer); + + safeRead.Transfer (oldData, oldTypeTree.m_Name.c_str ()); + + temp.End (); + + data = oldData; + + return true; +} + +#define REGISTER_CONVERTER(from, to) \ +SafeBinaryRead::RegisterConverter (SerializeTraits<from>::GetTypeString (NULL), SerializeTraits<to>::GetTypeString (NULL), \ + StdTemplateConversionFunction<from, to>) + +#endif +#endif diff --git a/Runtime/Serialize/SerializeTraits.h b/Runtime/Serialize/SerializeTraits.h new file mode 100644 index 0000000..9d5b946 --- /dev/null +++ b/Runtime/Serialize/SerializeTraits.h @@ -0,0 +1,533 @@ +#ifndef SERIALIZETRAITS_H +#define SERIALIZETRAITS_H + +#include "TypeTree.h" +#include "Runtime/Utilities/LogAssert.h" +#include "SerializeUtility.h" +#include "SerializationMetaFlags.h" +#include "Runtime/Utilities/vector_utility.h" +#include "Runtime/Utilities/vector_map.h" +#include "Runtime/Utilities/vector_set.h" +#include "Runtime/Utilities/dense_hash_map.h" +#include "Runtime/Utilities/dynamic_array.h" +#include <map> +#include <set> +#include <list> +#include <deque> +#include "SerializeTraitsBase.h" + +class SerializedFile; +class SafeBinaryRead; + +typedef void TransferTypelessCallback (UInt8* data, int byteSize, int instanceID, int userdata); + +/* + + You can use SerializeTraits to setup transfer functions for classes where you can't change to code, eg. the STL. + You might also want to use it when writing custom converters. + + template<> + class SerializeTraits<Vector4f> : public SerializeTraitsBase<Vector4f> + { + public: + + typedef Vector4f value_type; + + inline static const char* GetTypeString () { return value_type::GetTypeString (); } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + data.Transfer (transfer); + } + + template<class TransferFunction> + static void Convert (value_type& data, TransferFunction& transfer) + { + const TypeTree& oldTypeTree = transfer.GetActiveOldTypeTree (); + const std::string& oldType = transfer.GetActiveOldTypeTree ().m_Type; + if (oldType == "Vector3f") + { + Vector3f temp = data; + temp.Transfer (transfer); + data = temp; + return true; + } + else + return false; + } + + /// Returns whether or not a this type is to be treated as a seperate channel in the animation system + static bool IsAnimationChannel () { return T::IsAnimationChannel (); } + }; + +*/ + + +#define DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS(x) \ + inline static const char* GetTypeString (void* p = 0) { return #x; } \ + inline static bool IsAnimationChannel () { return true; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } + +template<> +struct SerializeTraits<float> : public SerializeTraitsBaseForBasicType<float> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (float) +}; + +template<> +struct SerializeTraits<double> : public SerializeTraitsBaseForBasicType<double> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (double) +}; + +template<> +struct SerializeTraits<SInt32> : public SerializeTraitsBaseForBasicType<SInt32> +{ + // We use "int" rather than "SInt32" here for backwards-compatibility reasons. + // "SInt32" and "int" used to be two different types (as were "UInt32" and "unsigned int") + // that we now serialize through same path. We use "int" instead of "SInt32" as the common + // identifier as it was more common. + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (int) +}; + +template<> +struct SerializeTraits<UInt32> : public SerializeTraitsBaseForBasicType<UInt32> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (unsigned int) // See definition of "int" above. +}; + +template<> +struct SerializeTraits<SInt64> : public SerializeTraitsBaseForBasicType<SInt64> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt64) +}; +template<> +struct SerializeTraits<UInt64> : public SerializeTraitsBaseForBasicType<UInt64> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt64) +}; + +template<> +struct SerializeTraits<SInt16> : public SerializeTraitsBaseForBasicType<SInt16> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt16) +}; + +template<> +struct SerializeTraits<UInt16> : public SerializeTraitsBaseForBasicType<UInt16> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt16) +}; + +template<> +struct SerializeTraits<SInt8> : public SerializeTraitsBaseForBasicType<SInt8> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (SInt8) +}; + +template<> +struct SerializeTraits<UInt8> : public SerializeTraitsBaseForBasicType<UInt8> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (UInt8) +}; + +template<> +struct SerializeTraits<char> : public SerializeTraitsBaseForBasicType<char> +{ + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (char) +}; + +template<> +struct SerializeTraits<bool> : public SerializeTraitsBase<bool> +{ + typedef bool value_type; + DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL_TRAITS (bool) + + static int GetByteSize () { return 1; } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + #if (defined __ppc__) && !UNITY_WII + AssertIf (sizeof(bool) != 4); + UInt8& temp = *(reinterpret_cast<UInt8*>(&data) + 3); + + transfer.TransferBasicData (temp); + + // When running in debug mode in OS X (-O0 in gcc), + // bool values which are not exactly 0x01 are treated as false. + // We don't want this. Cast UInt8 to bool to fix this. + if (transfer.IsReading()) + data = temp; + #if DEBUGMODE + AssertIf((transfer.IsReading() || transfer.IsWriting()) && (reinterpret_cast<int&> (data) != 0 && reinterpret_cast<int&> (data) != 1)); + #endif + #else + AssertIf (sizeof(bool) != 1); + UInt8& temp = reinterpret_cast<UInt8&>(data); + transfer.TransferBasicData (temp); + + // When running in debug mode in OS X (-O0 in gcc), + // bool values which are not exactly 0x01 are treated as false. + // We don't want this. Cast UInt8 to bool to fix this. + #if DEBUGMODE + if (transfer.IsReading()) + data = temp; + // You constructor or Reset function is not setting the bool value to a defined value! + AssertIf((transfer.IsReading() || transfer.IsWriting()) && (temp != 0 && temp != 1)); + #endif + #endif + } +}; + + + +#define DEFINE_GET_TYPESTRING_MAP_CONTAINER(x) \ +inline static const char* GetTypeString (void*) { return #x; } \ +inline static bool IsAnimationChannel () { return false; } \ +inline static bool MightContainPPtr () { return SerializeTraits<FirstClass>::MightContainPPtr() || SerializeTraits<SecondClass>::MightContainPPtr(); } \ +inline static bool AllowTransferOptimization () { return false; } + +template<> +class SerializeTraits<UnityStr> : public SerializeTraitsBase<UnityStr> +{ +public: + + typedef UnityStr value_type; + inline static const char* GetTypeString (value_type* x = NULL) { return "string"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return false; } + inline static bool AllowTransferOptimization () { return false; } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data, kHideInEditorMask); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + + static void ResizeSTLStyleArray (value_type& data, int rs) + { + data.resize (rs, 1); + } + +}; + +// Do not add this serialization function. All serialized strings should use UnityStr instead of std::string +//template<class Traits, class Allocator> +//class SerializeTraits<std::basic_string<char,Traits,Allocator> > : public SerializeTraitsBase<std::basic_string<char,Traits,Allocator> > + +template<class T, class Allocator> +class SerializeTraits<std::vector<T, Allocator> > : public SerializeTraitsBase<std::vector<T, Allocator> > +{ + public: + + typedef std::vector<T,Allocator> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { resize_trimmed (data, rs); } +}; + +template<class Allocator> +class SerializeTraits<std::vector<UInt8,Allocator> > : public SerializeTraitsBase<std::vector<UInt8,Allocator> > +{ + public: + + typedef std::vector<UInt8,Allocator> value_type; + + inline static const char* GetTypeString (void* x = NULL) { return "vector"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return false; } + inline static bool AllowTransferOptimization () { return false; } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { resize_trimmed (data, rs); } +}; + +template<class T, class Allocator> +class SerializeTraits<std::list<T,Allocator> > : public SerializeTraitsBase<std::list<T,Allocator> > +{ + public: + + typedef std::list<T,Allocator> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return false; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize (rs); } +}; + +template<class FirstClass, class SecondClass> +class SerializeTraits<std::pair<FirstClass, SecondClass> > : public SerializeTraitsBase<std::pair<FirstClass, SecondClass> > +{ + public: + + typedef std::pair<FirstClass, SecondClass> value_type; + inline static const char* GetTypeString (void* x = NULL) { return "pair"; } + inline static bool IsAnimationChannel () { return false; } + inline static bool MightContainPPtr () { return SerializeTraits<FirstClass>::MightContainPPtr() || SerializeTraits<SecondClass>::MightContainPPtr(); } +// inline static bool AllowTransferOptimization () { return SerializeTraits<FirstClass>::AllowTransferOptimization() || SerializeTraits<SecondClass>::AllowTransferOptimization(); } + inline static bool AllowTransferOptimization () { return false; } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.Transfer (data.first, "first"); + transfer.Transfer (data.second, "second"); + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class SerializeTraits<std::map<FirstClass, SecondClass, Compare, Allocator> > : public SerializeTraitsBase<std::map<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::map<FirstClass, SecondClass, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template<class FirstClass, class SecondClass, class HashFunction, class Compare, class Allocator> +class SerializeTraits<dense_hash_map<FirstClass, SecondClass, HashFunction, Compare, Allocator> > : public SerializeTraitsBase<dense_hash_map<FirstClass, SecondClass, HashFunction, Compare, Allocator> > +{ + public: + + typedef dense_hash_map<FirstClass, SecondClass, HashFunction, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class SerializeTraits<std::multimap<FirstClass, SecondClass, Compare, Allocator> > : public SerializeTraitsBase<std::multimap<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::multimap<FirstClass, SecondClass, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER(map) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + + +template<class T, class Compare, class Allocator> +class SerializeTraits<std::set<T, Compare, Allocator> > : public SerializeTraitsBase<std::set<T, Compare, Allocator> > +{ + public: + + typedef std::set<T, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class SerializeTraits<vector_map<FirstClass, SecondClass, Compare, Allocator> > : public SerializeTraitsBase<vector_map<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef vector_map<FirstClass, SecondClass, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER (map) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + + + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class SerializeTraits<us_vector_map<FirstClass, SecondClass, Compare, Allocator> > : public SerializeTraitsBase<vector_map<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef vector_map<FirstClass, SecondClass, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_MAP_CONTAINER (map) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + +template<class T, class Compare, class Allocator> +class SerializeTraits<vector_set<T, Compare, Allocator> > : public SerializeTraitsBase<vector_set<T, Compare, Allocator> > +{ + public: + + typedef vector_set<T, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + +template<class T, class Compare, class Allocator> +class SerializeTraits<us_vector_set<T, Compare, Allocator> > : public SerializeTraitsBase<vector_set<T, Compare, Allocator> > +{ + public: + + typedef vector_set<T, Compare, Allocator> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (set) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.get_vector ().resize (rs); } +}; + + +// Vector<bool> serialization is not allowed +template<class Allocator> +class SerializeTraits<std::vector<bool, Allocator> > : public SerializeTraitsBase<std::vector<bool, Allocator> > +{ + public: + // disallow vector<bool> serialization +}; + + +template<class T, size_t align> +class SerializeTraits<dynamic_array<T, align> > : public SerializeTraitsBase<dynamic_array<T, align> > +{ +public: + + typedef dynamic_array<T, align> value_type; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize_initialized(rs); } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + data.assign_external(reinterpret_cast<T*> (begin), reinterpret_cast<T*> (end)); + } +}; + +template<> +class SerializeTraits<dynamic_array<UInt8> > : public SerializeTraitsBase<dynamic_array<UInt8> > +{ +public: + + typedef dynamic_array<UInt8> value_type; + typedef UInt8 T; + DEFINE_GET_TYPESTRING_CONTAINER (vector) + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferSTLStyleArray (data); + transfer.Align(); + } + + static bool IsContinousMemoryArray () { return true; } + static void ResizeSTLStyleArray (value_type& data, int rs) { data.resize_initialized(rs); } + + static void resource_image_assign_external (value_type& data, void* begin, void* end) + { + data.assign_external(reinterpret_cast<UInt8*> (begin), reinterpret_cast<UInt8*> (end)); + } +}; + + +template<class T> +struct NonConstContainerValueType +{ + typedef typename T::value_type value_type; +}; + +template<class T> +struct NonConstContainerValueType<std::set<T> > +{ + typedef T value_type; +}; + +template<class T0, class T1, class Compare, class Allocator> +struct NonConstContainerValueType<std::map<T0, T1, Compare, Allocator> > +{ + typedef std::pair<T0, T1> value_type; +}; + +template<class T0, class T1, class Compare, class Allocator> +struct NonConstContainerValueType<std::multimap<T0, T1, Compare, Allocator> > +{ + typedef std::pair<T0, T1> value_type; +}; + +template<class T0, class T1, class HashFunction, class Compare, class Allocator> +struct NonConstContainerValueType<dense_hash_map<T0, T1, HashFunction, Compare, Allocator> > +{ + typedef std::pair<T0, T1> value_type; +}; + +#endif diff --git a/Runtime/Serialize/SerializeTraitsBase.h b/Runtime/Serialize/SerializeTraitsBase.h new file mode 100644 index 0000000..b3a85fd --- /dev/null +++ b/Runtime/Serialize/SerializeTraitsBase.h @@ -0,0 +1,61 @@ +#pragma once + +template<class T> +class SerializeTraitsBase +{ + public: + + typedef T value_type; + + static int GetByteSize () {return sizeof (value_type);} + static size_t GetAlignOf() {return ALIGN_OF(value_type);} + + static void resource_image_assign_external (value_type& /*data*/, void* /*begin*/, void* /*end*/) + { + AssertString("Unsupported"); + } +}; + +template<class T> +class SerializeTraitsBaseForBasicType : public SerializeTraitsBase<T> +{ +public: + typedef T value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + transfer.TransferBasicData (data); + } +}; + +template<class T> +class SerializeTraits : public SerializeTraitsBase<T> +{ +public: + + typedef T value_type; + + inline static const char* GetTypeString (void* /*ptr*/) { return value_type::GetTypeString (); } + inline static bool MightContainPPtr () { return value_type::MightContainPPtr (); } + /// Returns whether or not a this type is to be treated as a seperate channel in the animation system + static bool IsAnimationChannel () { return T::IsAnimationChannel (); } + + /// AllowTransferOptimization can be used for type that have the same memory format as serialized format. + /// Eg. a float or a Vector3f. + /// StreamedBinaryRead will collapse the read into a direct read when reading an array with values that have AllowTransferOptimization. + static bool AllowTransferOptimization () { return T::AllowTransferOptimization (); } + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + data.Transfer (transfer); + } + +}; + +#define DEFINE_GET_TYPESTRING_CONTAINER(x) \ +inline static const char* GetTypeString (void*) { return #x; } \ +inline static bool IsAnimationChannel () { return false; } \ +inline static bool MightContainPPtr () { return SerializeTraits<T>::MightContainPPtr(); } \ +inline static bool AllowTransferOptimization () { return false; } diff --git a/Runtime/Serialize/SerializeUtility.h b/Runtime/Serialize/SerializeUtility.h new file mode 100644 index 0000000..086f0e3 --- /dev/null +++ b/Runtime/Serialize/SerializeUtility.h @@ -0,0 +1,95 @@ +#ifndef SERIALIZEUTILITY_H +#define SERIALIZEUTILITY_H + + +#include "SerializationMetaFlags.h" + +#define TRANSFER(x) transfer.Transfer (x, #x) +#define TRANSFER_SIMPLE(x) transfer.Transfer (x, #x, kSimpleEditorMask) + +#if UNITY_EDITOR +#define TRANSFER_EDITOR_ONLY(x) if (!transfer.IsSerializingForGameRelease()) { transfer.Transfer (x, #x, kDontAnimate); } +#else +#define TRANSFER_EDITOR_ONLY(x) { } +#endif + +#if UNITY_EDITOR +#define TRANSFER_EDITOR_ONLY_HIDDEN(x) if (!transfer.IsSerializingForGameRelease()) { transfer.Transfer (x, #x, kHideInEditorMask); } +#else +#define TRANSFER_EDITOR_ONLY_HIDDEN(x) { } +#endif + +#define TRANSFER_WITH_CUSTOM_GET_SET(TYPE, STR_NAME, GET, SET, OPTIONS) \ + { \ + TYPE value; \ + if (transfer.IsWriting ()) { GET ; } \ + transfer.Transfer(value, STR_NAME, OPTIONS); \ + if (transfer.DidReadLastProperty ()) { SET ; } \ + } + +#define TRANSFER_PROPERTY(TYPE,NAME,GET,SET)\ + TRANSFER_WITH_CUSTOM_GET_SET(TYPE, #NAME, value = GET (), SET (value), kNoTransferFlags) + +#define TRANSFER_ENUM(x) { Assert(sizeof(x) == sizeof(int)); transfer.Transfer ((int&)x, #x); } + +#if UNITY_EDITOR +#define TRANSFER_DEBUG(x) { if (transfer.GetFlags () & kSerializeDebugProperties) transfer.Transfer (x, #x, kDebugPropertyMask | kNotEditableMask); } +#else +#define TRANSFER_DEBUG(x) +#endif + +template<class T> +inline bool SerializePrefabIgnoreProperties (T& transfer) +{ + return (transfer.GetFlags() & kSerializeForPrefabSystem) == 0; +} + +/// Usage: TRANSFER_PROPERTY_DEBUG(bool, m_Enabled, data->GetEnabled) +#define TRANSFER_PROPERTY_DEBUG(TYPE,NAME,GET) \ +if (transfer.GetFlags () & kSerializeDebugProperties){\ + TYPE NAME;\ + if (transfer.IsWriting ())\ + NAME = GET ();\ + transfer.Transfer (NAME, #NAME, kDebugPropertyMask | kNotEditableMask);\ +} + + +#define DEFINE_GET_TYPESTRING(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return true; } \ + inline static bool AllowTransferOptimization () { return false; } + +#define DEFINE_GET_TYPESTRING_IS_ANIMATION_CHANNEL(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return true; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } + + +#define DECLARE_SERIALIZE(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return true; } \ + inline static bool AllowTransferOptimization () { return false; } \ + template<class TransferFunction> \ + void Transfer (TransferFunction& transfer); + +#define DECLARE_SERIALIZE_NO_PPTR(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return false; } \ + template<class TransferFunction> \ + void Transfer (TransferFunction& transfer); + +#define DECLARE_SERIALIZE_OPTIMIZE_TRANSFER(x) \ + inline static const char* GetTypeString () { return #x; } \ + inline static bool IsAnimationChannel () { return false; } \ + inline static bool MightContainPPtr () { return false; } \ + inline static bool AllowTransferOptimization () { return true; } \ + template<class TransferFunction> \ + void Transfer (TransferFunction& transfer); + + +#endif diff --git a/Runtime/Serialize/SerializedFile.cpp b/Runtime/Serialize/SerializedFile.cpp new file mode 100644 index 0000000..26a0a3b --- /dev/null +++ b/Runtime/Serialize/SerializedFile.cpp @@ -0,0 +1,1520 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" +#include "SerializedFile.h" +#include "Runtime/Utilities/Utility.h" +#include "SerializeConversion.h" +#include "TransferUtility.h" +#include "Runtime/Serialize/TransferFunctions/TransferNameConversions.h" +#include "CacheWrap.h" +#include "Runtime/Utilities/Word.h" +#include "Configuration/UnityConfigureVersion.h" +#include "BuildTargetVerification.h" +#include "Runtime/Utilities/FileUtilities.h" +#if UNITY_WII +#include "PlatformDependent/Wii/WiiUtility.h" +#include "PlatformDependent/Wii/WiiLoadingScreen.h" +#endif + +#include "Runtime/Misc/Allocator.h" + +/// Set this to 1 to dump type trees to the console when they don't match between +/// the runtime and the file that is being loaded. This is most useful when debugging +/// loading issues in players (also enable DEBUG_FORCE_ALWAYS_WRITE_TYPETREES in +/// BuildPlayerUtility.cpp to have type trees included in player data). +#define DEBUG_LOG_TYPETREE_MISMATCHES 0 + +using namespace std; + +enum { kCurrentSerializeVersion = 9 }; + +const char* kAssetBundleVersionNumber = "1"; +const char* kUnityTextMagicString = "%YAML 1.1"; +#define kUnityTextHeaderFileID -1 + +bool IsSerializedFileTextFile(string pathName) +{ + const int magiclen = strlen(kUnityTextMagicString); + + char compare[256]; + if (!ReadFromFile (pathName, compare, 0, magiclen)) + return false; + + compare[magiclen] = '\0'; + if (strcmp(compare, kUnityTextMagicString) == 0) + return true; + + return false; +} + + +#if ENABLE_SECURITY +#define TEST_LEN(x) if (iterator + sizeof(x) > end) \ +{\ + return false; \ +} + +#define TEST_READ_SIZE(x) if (iterator + x > end) \ +{\ +return false; \ +} + +#else +#define TEST_LEN(x) +#define TEST_READ_SIZE(x) +#endif + +static const int kHeaderSize_Ver8 = 12; +static const int kPreallocateFront = 4096; + +struct SerializedFileHeader +{ + // This header is always in BigEndian when in file + // Metadata follows directly after the header + UInt32 m_MetadataSize; + UInt32 m_FileSize; + UInt32 m_Version; + UInt32 m_DataOffset; + UInt8 m_Endianess; + UInt8 m_Reserved[3]; + + void SwapEndianess () + { + SwapEndianBytes (m_MetadataSize); + SwapEndianBytes (m_FileSize); + SwapEndianBytes (m_Version); + SwapEndianBytes (m_DataOffset); + } +}; + +int RemapClassIDToNewClassID (int classID) +{ + switch(classID) + { + case 1012: return 1011; // AvatarSkeletonMask -> AvatarMask + default: return classID; + } +} + +SerializedFile::SerializedFile () +: m_Externals(1024,kMemSerialization) +{ + m_ReadOffset = 0; + m_WriteDataOffset = 0; + m_IsDirty = false; + m_MemoryStream = false; + m_HasErrors = false; + m_CachedFileStream = false; + m_TargetPlatform = kBuildNoTargetPlatform; + m_SubTarget = 0; + + #if SUPPORT_TEXT_SERIALIZATION + m_IsTextFile = false; + #endif + + #if SUPPORT_SERIALIZE_WRITE + m_CachedWriter = NULL; + #endif + + m_ReadFile = NULL; +} + + +#if SUPPORT_SERIALIZE_WRITE +bool SerializedFile::InitializeWrite (CachedWriter& cachedWriter, BuildTargetSelection target, int options) +{ + SET_ALLOC_OWNER(this); + m_TargetPlatform = target.platform; + m_SubTarget = target.subTarget; + + m_CachedWriter = &cachedWriter; + + Assert (!((options & kAllowTextSerialization) && (options & kSerializeGameRelease))); + m_IsTextFile = options & kAllowTextSerialization; + + if (!m_IsTextFile) + { + void* buffer = alloca (kPreallocateFront); + memset (buffer, 0, kPreallocateFront); + + // Write header and reserve space for metadata. In case the resulting metadata will not fit + // in the preallocated space we'll remove it and write it tightly packed in FinalizeWrite later. + // In case it fits, we'll have a hole between meta and object data and that's fine. + + m_CachedWriter->Write(buffer, kPreallocateFront); + m_WriteDataOffset = m_CachedWriter->GetPosition (); + } + + return FinalizeInit(options); +} +#endif + +bool SerializedFile::InitializeRead (const string& path, ResourceImageGroup& resourceImage, unsigned cacheSize, unsigned cacheCount, int options, int readOffset) +{ + SET_ALLOC_OWNER(this); + m_ReadOffset = readOffset; + m_ReadFile = UNITY_NEW( FileCacherRead (path, cacheSize, cacheCount), kMemFile); + m_ResourceImageGroup = resourceImage; + + return FinalizeInit(options); +} + +bool SerializedFile::InitializeMemoryBlocks (const string& path, UInt8** buffer, unsigned size, unsigned offset, int options) +{ + SET_ALLOC_OWNER(this); + m_MemoryStream = true; + m_ReadOffset = offset; + m_ReadFile = UNITY_NEW( MemoryCacherReadBlocks (buffer, size, kCacheBlockSize), kMemFile); + + return FinalizeInit(options); +} + +bool SerializedFile::FinalizeInit (int options) +{ + m_Options = options; + #if GAMERELEASE + m_Options |= kSerializeGameRelease; + #endif + + if (m_Options & kSwapEndianess) + m_FileEndianess = kOppositeEndianess; + else + m_FileEndianess = kActiveEndianess; + + if (m_ReadFile) + { +#if SUPPORT_TEXT_SERIALIZATION + const int magiclen = strlen(kUnityTextMagicString); + char compare[256]; + if (m_ReadFile->GetFileLength () >= magiclen) + { + ReadFileCache (*m_ReadFile, compare, 0 + m_ReadOffset, magiclen); + compare[magiclen] = '\0'; + if (strcmp(compare, kUnityTextMagicString) == 0) + { + m_IsTextFile = true; + m_ReadOffset += magiclen; + return ReadHeaderText(); + } + } + m_IsTextFile = false; +#endif + return ReadHeader(); + } + else + { +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + string label = kUnityTextMagicString; + label += "\n%TAG !u! tag:unity3d.com,2011:\n"; + m_CachedWriter->Write (&label[0], label.length()); + } +#endif + return true; +} +} + +SerializedFile::~SerializedFile () +{ + UNITY_DELETE( m_ReadFile, kMemFile); +} + + +#if SUPPORT_SERIALIZE_WRITE +bool SerializedFile::FinishWriting () +{ + AssertIf(m_CachedWriter == NULL); + + if (m_CachedWriter != NULL) + { + if (!m_IsTextFile) + { + SerializationCache metadataBuffer; + + if (!ShouldSwapEndian()) + { + BuildMetadataSection<false> (metadataBuffer, m_WriteDataOffset); + return WriteHeader<false> (metadataBuffer); + } + else + { + BuildMetadataSection<true> (metadataBuffer, m_WriteDataOffset); + return WriteHeader<true> (metadataBuffer); + } + } + else + { + bool success = m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase().WriteHeaderAndCloseFile(NULL, 0, 0); + return success; + } + } + + return false; +} + +static void WriteAlignmentData (File& file, size_t misalignment) +{ + Assert (misalignment < SerializedFile::kSectionAlignment); + UInt8 data[SerializedFile::kSectionAlignment]; + memset (data, 0, misalignment); + file.Write(data, misalignment); +} + +template<bool kSwap> +bool SerializedFile::WriteHeader (SerializationCache& metadata) +{ + bool success = true; + + // The aggregated metadata fits into the pre-written block, so write it directly. + if (metadata.size () <= kPreallocateFront - sizeof (SerializedFileHeader)) + { + UInt8* temp = (UInt8*)alloca (kPreallocateFront); + + SerializedFileHeader& header = *(SerializedFileHeader*)temp; + header.m_MetadataSize = metadata.size (); + header.m_FileSize = m_CachedWriter->GetPosition (); + header.m_Version = kCurrentSerializeVersion; + header.m_DataOffset = m_WriteDataOffset; + header.m_Endianess = m_FileEndianess; + memset (header.m_Reserved, 0, sizeof header.m_Reserved); + + if (kActiveEndianess != kBigEndian) + header.SwapEndianess (); + + std::copy (metadata.begin (), metadata.end (), temp + sizeof (SerializedFileHeader)); + success &= m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase ().WriteHeaderAndCloseFile (temp, 0, sizeof (SerializedFileHeader) + metadata.size ()); + } + else + { + // metadata doesn't fit, therefore close the file, write header + metadata to another file + // and copy data over from 'this' one. + + success &= m_CachedWriter->CompleteWriting(); + success &= m_CachedWriter->GetCacheBase ().WriteHeaderAndCloseFile (NULL, 0, 0); + + size_t dataFileSize = m_CachedWriter->GetPosition (); + if (dataFileSize < kPreallocateFront) + return false; + + size_t dataSize = dataFileSize - kPreallocateFront; + size_t dataOffsetOriginal = sizeof (SerializedFileHeader) + metadata.size (); + size_t dataOffset = RoundUp (dataOffsetOriginal, kSectionAlignment); + + std::string originalPath = m_CachedWriter->GetCacheBase().GetPathName (); + std::string tempPath = GenerateUniquePathSafe (originalPath); + + SerializedFileHeader header = + { + metadata.size (), dataOffset + dataSize, + kCurrentSerializeVersion, + dataOffset, + m_FileEndianess, 0, 0, 0 + }; + + if (kActiveEndianess != kBigEndian) + header.SwapEndianess (); + + File file; + success &= file.Open(tempPath, File::kWritePermission); + + // header + success &= file.Write (&header, sizeof (header)); + + // metadata + success &= file.Write (&*metadata.begin (), metadata.size ()); + if (dataOffset != dataOffsetOriginal) + WriteAlignmentData (file, dataOffset - dataOffsetOriginal); + FatalErrorIf (dataOffset != file.GetPosition ()); + + { + enum { kCopyChunck = 1 * 1024 * 1024 }; + + UInt8* buffer; + ALLOC_TEMP(buffer, UInt8, kCopyChunck); + + File srcFile; + success &= srcFile.Open(originalPath, File::kReadPermission); + + size_t position = kPreallocateFront; + size_t left = dataSize; + while (left > 0 && success) + { + size_t toRead = (std::min)((size_t)kCopyChunck, left); + int wasRead = srcFile.Read (position, buffer, toRead); + success &= file.Write (buffer, wasRead); + position += toRead; + left -= toRead; + } + success &= srcFile.Close (); + + success &= file.Close (); + } + + // move the temp file over to the destination + success &= DeleteFile(originalPath); + success &= MoveFileOrDirectory (tempPath, originalPath); + } + + return success; +} +#endif // SUPPORT_SERIALIZE_WRITE + +enum { kMaxTypeCount = 100000 }; + +bool SerializedFile::ReadHeader () +{ + AssertIf (m_ReadFile == NULL); + + SerializedFileHeader header; + + if (m_ReadFile->GetFileLength () < sizeof (header)) + return false; + + ReadFileCache (*m_ReadFile, &header, m_ReadOffset, sizeof (header)); + + if (kActiveEndianess == kLittleEndian) + header.SwapEndianess (); + + // Consistency check if the file is a valid serialized file. + if (header.m_MetadataSize == -1) + return false; + if (header.m_Version == 1) + return false; + if (header.m_Version > kCurrentSerializeVersion) + return false; + + unsigned metadataSize, metadataOffset; + unsigned dataSize, dataOffset; + unsigned dataEnd; + + if (header.m_Version >= 9) + { + // If we're reading a stream file, m_ReadOffset + header.m_FileSize will not necessarilly be equal to m_ReadFile->GetFileLength(), + // because there can be few padding bytes which doesn't count into header.m_FileSize + // See WriteStreamFile in BuildPlayerUtility.cpp + if ((m_ReadOffset + header.m_FileSize) > m_ReadFile->GetFileLength () || header.m_DataOffset > header.m_FileSize) + return false; + + // [header][metadata[...]][data] + + metadataOffset = sizeof header; + metadataSize = header.m_MetadataSize; + + m_FileEndianess = header.m_Endianess; + + dataOffset = header.m_DataOffset; + dataSize = header.m_FileSize - header.m_DataOffset; + dataEnd = dataOffset + dataSize; + } + else + { + // [header][data][metadata] + + // We set dataOffset to zero, because offsets in object table are file-start based + dataOffset = 0; + dataSize = header.m_FileSize - header.m_MetadataSize - kHeaderSize_Ver8; + dataEnd = header.m_FileSize - header.m_MetadataSize; + + // Offset by one, because we're reading the endianess flag right here + metadataOffset = header.m_FileSize - header.m_MetadataSize + 1; + metadataSize = header.m_MetadataSize - 1; + + if (metadataSize == -1 || (m_ReadOffset + header.m_FileSize) > m_ReadFile->GetFileLength () || dataEnd > header.m_FileSize) + return false; + + ReadFileCache (*m_ReadFile, &m_FileEndianess, m_ReadOffset + metadataOffset - 1, sizeof (m_FileEndianess)); + } + + // Check endianess validity + if (m_FileEndianess != kBigEndian && m_FileEndianess != kLittleEndian) + return false; + + SerializationCache metadataBuffer; + metadataBuffer.resize (metadataSize); + ReadFileCache (*m_ReadFile, &metadataBuffer[0], m_ReadOffset + metadataOffset, metadataSize); + + bool result; + if (m_FileEndianess == kActiveEndianess) + { + result = ReadMetadata<false>(header.m_Version, dataOffset, &*metadataBuffer.begin (), metadataBuffer.size (), dataEnd); + } + else + { + result = ReadMetadata<true>(header.m_Version, dataOffset, &*metadataBuffer.begin (), metadataBuffer.size (), dataEnd); + } + + if (!result) + { + ErrorString(Format("Failed to read file '%s' because it is corrupted.", m_ReadFile->GetPathName ().c_str())); + } + return result; +} + +#if SUPPORT_TEXT_SERIALIZATION +bool SerializedFile::IndexTextFile() +{ + const size_t kBufferLength = 1024; + const size_t kMaxLineLength = 256; + bool hasMergeConflicts = false; + string read; + read.resize (kBufferLength); + + size_t readPos = 0; + size_t lineStart = 0; + int lineCount = 1; //We start counting lines at one, not zero. + ObjectInfo *curInfo = NULL; + + const char *guidLabel = "guid: "; + int guidLabelLen = strlen (guidLabel); + int guidLabelPos = 0; + bool lineContainsGUID = false; + std::string prevLine; + std::set <FileIdentifier> externals; + while (readPos < m_ReadFile->GetFileLength() - m_ReadOffset) + { + size_t readBufferLength = std::min (m_ReadFile->GetFileLength() - m_ReadOffset - readPos, kBufferLength); + ReadFileCache (*m_ReadFile, &read[0], m_ReadOffset + readPos, readBufferLength); + + for (size_t i=0; i<readBufferLength; i++) + { + if (read[i] == guidLabel[guidLabelPos]) + { + guidLabelPos++; + if (guidLabelPos == guidLabelLen) + lineContainsGUID = true; + } + else + guidLabelPos = 0; + + if (read[i] == '\n') + { + lineCount++; + size_t lineEnd = i+1+readPos; + size_t lineLength = std::min(lineEnd - lineStart, kMaxLineLength); + string line; + if (lineStart < readPos) + { + line.resize (lineLength); + ReadFileCache (*m_ReadFile, &line[0], m_ReadOffset + lineStart, lineLength); + } + else + line = read.substr (lineStart - readPos, lineLength); + + if (line.length()) + { + switch (line[0]) + { + case ' ': + break; //fast path for most common case. + case '-': + { + if (curInfo) + curInfo->byteSize = lineStart - curInfo->byteStart; + SInt32 fileID, classID; + if (sscanf(line.c_str(), "--- !u!%d &%d", (int*)&classID, (int*)&fileID) == 2) + { + curInfo = &m_Object[fileID]; + curInfo->classID = RemapClassIDToNewClassID(classID); + curInfo->typeID = 0; + curInfo->byteStart = lineEnd; + curInfo->isDestroyed = false; + curInfo->debugLineStart = lineCount; + } + } + break; + #if UNITY_EDITOR + case '>': + case '<': + case '=': + if (!hasMergeConflicts) + WarningStringMsg ("The file %s seems to have merge conflicts. Please open it in a text editor and fix the merge.\n", m_DebugPath.c_str()); + hasMergeConflicts = true; + break; + #endif + } + if (lineContainsGUID) + { + if (line.find ('}') == string::npos) + prevLine = line; + else + { + line = prevLine + line; + line = line.substr(line.find ('{')); + YAMLRead read (line.c_str(), line.size(), 0, &m_DebugPath, lineCount-1); + + FileIdentifier id; + + read.Transfer (id.guid, "guid"); + read.Transfer (id.type, "type"); + id.Fix_3_5_BackwardsCompatibility(); + + if (id.guid != UnityGUID()) + externals.insert (id); + else + ErrorStringMsg ("Could not extract GUID in text file %s at line %d.", m_DebugPath.c_str(), lineCount-1); + + lineContainsGUID = false; + prevLine = ""; + } + } + } + lineStart = lineEnd; + } + } + + readPos += readBufferLength; + } + if (curInfo) + curInfo->byteSize = readPos - curInfo->byteStart; + + m_Externals.assign (externals.begin(), externals.end()); + return !hasMergeConflicts; +} + +bool SerializedFile::ReadHeaderText () +{ + Assert (m_ReadFile != NULL); + return IndexTextFile (); +} + +template<class T> +void SerializedFile::WriteTextSerialized (std::string &label, T &data, int options) +{ + Assert (m_CachedWriter != NULL); + + m_CachedWriter->Write (&label[0], label.length()); + + YAMLWrite writeStream (options, &m_DebugPath); + data.VirtualRedirectTransfer (writeStream); + writeStream.OutputToCachedWriter(m_CachedWriter); + if (writeStream.HasError()) + m_HasErrors = true; +} + +#endif + +// The header is put at the end of and is only allowed to be read at startup + +template<bool kSwap> +bool SerializedFile::ReadMetadata (int version, unsigned dataOffset, UInt8 const* data, size_t length, size_t dataFileEnd) +{ + AssertIf(kSwap && kActiveEndianess == m_FileEndianess); + AssertIf(!kSwap && kOppositeEndianess == m_FileEndianess); + SET_ALLOC_OWNER(this); + + UInt8 const* iterator = data, *end = data + length; + + // Read Unity version file was built with + UnityStr unityVersion; + if (version >= 7) + { + if (!ReadString(unityVersion, iterator, end)) + return false; + } + + // Build target platform verification + if (version >= 8) + { + TEST_LEN(m_TargetPlatform); + ReadHeaderCache<kSwap> (m_TargetPlatform, iterator); + + if (!CanLoadFileBuiltForTargetPlatform(static_cast<BuildTargetPlatform>(m_TargetPlatform))) + { + ErrorStringMsg( + "The file can not be loaded because it was created for another build target that is not compatible with this platform.\n" + "Please make sure to build asset bundles using the build target platform that it is used by.\n" + "File's Build target is: %d\n", + (int)m_TargetPlatform + ); + return false; + } + } + + // Read number of types + SInt32 typeCount; + TEST_LEN(typeCount); + ReadHeaderCache<kSwap> (typeCount, iterator); + + #if SUPPORT_SERIALIZED_TYPETREES + // Read types + for (int i=0;i<typeCount;i++) + { + TypeTree* readType = UNITY_NEW (TypeTree, kMemTypeTree); + TypeMap::key_type classID; + TEST_LEN(classID); + ReadHeaderCache<kSwap> (classID, iterator); + if (!ReadTypeTree (*readType, iterator, end, version, kSwap)) + return false; + classID = RemapClassIDToNewClassID(classID); + + m_Type[classID].SetOldType (readType); + } + #else + if (typeCount != 0) + { + ErrorString("Serialized file contains typetrees but the target can't use them. Will ignore typetrees."); + } + #endif + + SInt32 bigIDEnabled = 0; + if (version >= 7) + ReadHeaderCache<kSwap> (bigIDEnabled, iterator); + + // Read number of objects + SInt32 objectCount; + TEST_LEN(objectCount); + ReadHeaderCache<kSwap> (objectCount, iterator); + + // Check if the size is roughly out of bounds, we only want to prevent running out of memory due to insane objectCount value here. + TEST_READ_SIZE(objectCount * 12) + + // Read Objects + m_Object.reserve(objectCount); + for (int i=0;i<objectCount;i++) + { + LocalIdentifierInFileType fileID; + ObjectMap::mapped_type value; + + if (bigIDEnabled) + { +// AssertIf(fileID64 > LOCAL_IDENTIFIER_IN_FILE_SIZE); + UInt64 fileID64; + TEST_LEN(fileID64); + ReadHeaderCache<kSwap> (fileID64, iterator); + fileID = fileID64; + } + else + { + UInt32 fileID32; + TEST_LEN(fileID32); + ReadHeaderCache<kSwap> (fileID32, iterator); + fileID = fileID32; + } + + TEST_LEN(value); + ReadHeaderCache<kSwap> (value.byteStart, iterator); + ReadHeaderCache<kSwap> (value.byteSize, iterator); + ReadHeaderCache<kSwap> (value.typeID, iterator); + ReadHeaderCache<kSwap> (value.classID, iterator); + ReadHeaderCache<kSwap> (value.isDestroyed, iterator); + + value.byteStart += dataOffset; + + // TODO check this with joachim + value.typeID = RemapClassIDToNewClassID(value.typeID); + value.classID = RemapClassIDToNewClassID(value.classID); + + AssertIf (value.byteStart + value.byteSize > dataFileEnd); + if (value.byteStart < 0 || value.byteSize < 0 || value.byteStart + value.byteSize < value.byteStart || value.byteStart + value.byteSize > dataFileEnd) + return false; + + m_Object.push_unsorted(fileID, value); + //printf_console ("fileID: %d byteStart: %d classID: %d \n", fileID, value.byteStart, value.classID); + } + + // If there's no type tree then Unity version must mach exactly. + // + // For asset bundles we write the asset bundle serialize version and compare against that. + // The asset bundle itself contains hashes of all serialized classes and uses it to figure out if an asset bundle can be loaded. + bool needsVersionCheck = !m_Object.empty() && typeCount == 0 && (m_Options & kIsBuiltinResourcesFile) == 0; + if (needsVersionCheck) + { + bool versionPasses; + string::size_type newLinePosition = unityVersion.find('\n'); + // Compare Unity version number + if (newLinePosition == string::npos) + versionPasses = unityVersion == UNITY_VERSION; + // Compare asset bundle serialize version + else + versionPasses = string (unityVersion.begin() + newLinePosition + 1, unityVersion.end()) == kAssetBundleVersionNumber; + + if (!versionPasses) + { + ErrorStringMsg("Invalid serialized file version. File: \"%s\". Expected version: " UNITY_VERSION ". Actual version: %s.", m_ReadFile->GetPathName().c_str(), unityVersion.c_str()); + return false; + } + } + + #if SUPPORT_SERIALIZED_TYPETREES + if (unityVersion.find("3.5.0f5") == 0) + m_Options |= kWorkaround35MeshSerializationFuckup; + #endif + +// printf_console("file version: %s - '%s'\n", unityVersion.c_str(), m_DebugPath.c_str()); + + // Read externals/pathnames + SInt32 externalsCount; + TEST_LEN(externalsCount); + + ReadHeaderCache<kSwap> (externalsCount, iterator); + // Check if the size is roughly out of bounds, we only want to prevent running out of memory due to insane externalsCount value here. + TEST_READ_SIZE(externalsCount) + + m_Externals.resize (externalsCount); + + for (int i=0;i<externalsCount;i++) + { + if (version >= 5) + { + if (version >= 6) + { + ///@TODO: Remove from serialized file format + UnityStr tempEmpty; + if (!ReadString(tempEmpty, iterator, end)) + return false; + } + + TEST_LEN(m_Externals[i].guid.data); + for (int g=0;g<4;g++) + ReadHeaderCache<kSwap> (m_Externals[i].guid.data[g], iterator); + + TEST_LEN(m_Externals[i].type); + ReadHeaderCache<kSwap> (m_Externals[i].type, iterator); + if (!ReadString (m_Externals[i].pathName, iterator, end)) + return false; + } + else + { + if (!ReadString (m_Externals[i].pathName, iterator, end)) + return false; + } + + #if UNITY_EDITOR + m_Externals[i].Fix_3_5_BackwardsCompatibility (); + #endif + + m_Externals[i].CheckValidity (); + } + + // Read Userinfo string + if (version >= 5) + { + UnityStr userInformation; + if (!ReadString (userInformation, iterator, end)) + return false; + } + + Assert (iterator == end); + + return true; +} + +#if SUPPORT_SERIALIZE_WRITE + +template<bool kSwap> +void SerializedFile::BuildMetadataSection (SerializationCache& cache, unsigned dataOffsetInFile) +{ + // Write Unity version file is being built with + UnityStr version = UNITY_VERSION; + if (m_Options & kSerializedAssetBundleVersion) + { + version += "\n"; + version += kAssetBundleVersionNumber; + } + WriteString (version, cache); + + WriteHeaderCache<kSwap> (m_TargetPlatform, cache); + + if ((m_Options & kDisableWriteTypeTree) == 0) + { + // Write number of types + SInt32 typeCount = m_Type.size (); + WriteHeaderCache<kSwap> (typeCount, cache); + + // Write type data + for (TypeMap::iterator i = m_Type.begin ();i != m_Type.end ();i++) + { + AssertIf (i->second.GetOldType () == NULL); + WriteHeaderCache<kSwap> (i->first, cache); + WriteTypeTree (*i->second.GetOldType (), cache, kSwap); + } + } + else + { + SInt32 typeCount = 0; + WriteHeaderCache<kSwap> (typeCount, cache); + } + + SInt32 bigIDEnabled = LOCAL_IDENTIFIER_IN_FILE_SIZE > 32; + WriteHeaderCache<kSwap> (bigIDEnabled, cache); + + // Write number of objects + SInt32 objectCount = m_Object.size (); + WriteHeaderCache<kSwap> (objectCount, cache); + for (ObjectMap::iterator i = m_Object.begin ();i != m_Object.end ();i++) + { + if (bigIDEnabled) + { + UInt64 bigID = i->first; + WriteHeaderCache<kSwap> (bigID, cache); + } + else + { + UInt32 smallID = i->first; + WriteHeaderCache<kSwap> (smallID, cache); + } + + WriteHeaderCache<kSwap> (i->second.byteStart - dataOffsetInFile, cache); + WriteHeaderCache<kSwap> (i->second.byteSize, cache); + WriteHeaderCache<kSwap> (i->second.typeID, cache); + WriteHeaderCache<kSwap> (i->second.classID, cache); + WriteHeaderCache<kSwap> (i->second.isDestroyed, cache); + + //printf_console ("fileID: %d byteStart: %d classID: %d \n", i->first, i->second.byteStart, i->second.classID); + } + + // Write externals + objectCount = m_Externals.size (); + WriteHeaderCache<kSwap> (objectCount, cache); + for (int i=0;i<objectCount;i++) + { + UnityStr tempEmpty; + WriteString (tempEmpty, cache); + for (int g=0;g<4;g++) + WriteHeaderCache<kSwap> (m_Externals[i].guid.data[g], cache); + WriteHeaderCache<kSwap> (m_Externals[i].type, cache); + WriteString (m_Externals[i].pathName, cache); + } + + // Write User info + UnityStr tempUserInformation; + WriteString (tempUserInformation, cache); +} +#endif + +bool SerializedFile::IsAvailable (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + if (i == m_Object.end ()) + return false; + else + return ! i->second.isDestroyed; +} + +int SerializedFile::GetClassID (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.classID; +} + +int SerializedFile::GetByteStart (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.byteStart; +} + +int SerializedFile::GetByteSize (LocalIdentifierInFileType id) const +{ + ObjectMap::const_iterator i = m_Object.find (id); + AssertIf (i == m_Object.end ()); + return i->second.byteSize; +} +#if SUPPORT_SERIALIZED_TYPETREES +const TypeTree* SerializedFile::GetTypeTree (LocalIdentifierInFileType id) +{ + ObjectMap::iterator found = m_Object.find(id); + if (found == m_Object.end()) + return NULL; + + TypeMap::iterator type = m_Type.find (found->second.typeID); + if (type == m_Type.end ()) + return NULL; + return type->second.GetOldType (); +} +#endif + +#if !UNITY_EXTERNAL_TOOL +// objects: On return, all fileIDs to all objects in this Serialze +void SerializedFile::GetAllFileIDs (vector<LocalIdentifierInFileType>* objects)const +{ + AssertIf (objects == NULL); + + objects->reserve (m_Object.size ()); + ObjectMap::const_iterator i; + for (i=m_Object.begin ();i!=m_Object.end ();i++) + { + if (i->second.isDestroyed) + continue; + + Object::RTTI* rtti = Object::ClassIDToRTTI(i->second.classID); + if (rtti == NULL || rtti->factory == NULL) + continue; + + objects->push_back (i->first); + } +} +#endif +// objects: On return, all fileIDs to all objects in this Serialze +void SerializedFile::GetAllFileIDsUnchecked (vector<LocalIdentifierInFileType>* objects)const +{ + AssertIf (objects == NULL); + + objects->reserve (m_Object.size ()); + ObjectMap::const_iterator i; + for (i=m_Object.begin ();i!=m_Object.end ();i++) + { + if (i->second.isDestroyed) + continue; + + objects->push_back (i->first); + } +} + +LocalIdentifierInFileType SerializedFile::GetHighestID () const +{ + if (m_Object.empty ()) + return 0; + else + return m_Object.rbegin ()->first; +} + +// Returns whether or not the object referenced by id was destroyed +bool SerializedFile::DestroyObject (LocalIdentifierInFileType id) +{ + #if SUPPORT_SERIALIZE_WRITE + AssertIf(m_CachedWriter); + #endif + m_IsDirty = true; + + ObjectMap::iterator o; + o = m_Object.find (id); + if (o == m_Object.end ()) + { + SET_ALLOC_OWNER(this); + m_Object[id].isDestroyed = true; + return false; + } + else + { + o->second.isDestroyed = true; + return true; + } +} + +#if SUPPORT_SERIALIZE_WRITE + +void SerializedFile::WriteObject (Object& object, LocalIdentifierInFileType fileID, const BuildUsageTag& buildUsage) +{ + AssertIf (m_CachedWriter == NULL); + SET_ALLOC_OWNER(this); + + bool perObjectTypeTree = object.GetNeedsPerObjectTypeTree (); + + int typeID = object.GetClassID (); + + int mask = kNeedsInstanceIDRemapping | m_Options; + +#if UNITY_EDITOR + object.SetFileIDHint (fileID); +#endif + +#if SUPPORT_TEXT_SERIALIZATION + if (!m_IsTextFile) + { +#endif + + // Native C++ object typetrees share typetree by class id + if (!perObjectTypeTree) + { + // Include Type + Type& type = m_Type[typeID]; + + // If we need a perObjectTypeTree there is only one object using it (the one we are writing now) + // Thus we can safely replace the old typeTree + // If we have a per class typetree do not override the old typetree + if (type.GetOldType () == NULL) + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + GenerateTypeTree (object, typeTree, mask | kDontRequireAllMetaFlags); + + // Register the type tree + type.SetOldType (typeTree); + } + } + // Scripted objects we search the registered typetrees for duplicates and share + // or otherwise allocate a new typetree. + else + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + + GenerateTypeTree (object, typeTree, mask | kDontRequireAllMetaFlags); + + typeID = 0; + + // Find if there + for (TypeMap::iterator i=m_Type.begin();i!=m_Type.end();i++) + { + if (i->first > 0) + break; + + if (IsStreamedBinaryCompatbile(*typeTree, *i->second.GetOldType())) + { + typeID = i->first; + UNITY_DELETE(typeTree, kMemTypeTree); + break; + } + } + + // Allocate type id + if (typeID == 0) + { + if (m_Type.empty()) + typeID = -1; + else if (m_Type.begin()->first < 0) + typeID = m_Type.begin()->first - 1; + else + typeID = -1; + + Type& type = m_Type[typeID]; + + // Create new type and init it using a proxy transfer + // Register the type tree + type.SetOldType (typeTree); + } + } + +#if SUPPORT_TEXT_SERIALIZATION + } +#endif + + // We are not taking care of fragmentation. + const unsigned kFileAlignment = 8; + + unsigned unalignedByteStart = m_CachedWriter->GetPosition(); + + // Align the object to a kFileAlignment byte boundary + unsigned alignedByteStart = unalignedByteStart; + if (unalignedByteStart % kFileAlignment != 0) + alignedByteStart += kFileAlignment - unalignedByteStart % kFileAlignment; + + + AssertIf (m_Object.find (fileID) != m_Object.end () && m_Object.find (fileID)->second.classID != object.GetClassID ()); + ObjectInfo& info = m_Object[fileID]; + info.byteStart = alignedByteStart; + info.classID = object.GetClassID (); + info.isDestroyed = false; + info.typeID = typeID; + +/* ////// PRINT OUT serialized Data as ascii to console + if (false && gPrintfDataHack) + { + printf_console ("\n\nPrinting object: %d\n", fileID); + + // Set write marker to end of file and register the objects position in file + StreamedTextWrite writeStream; + CachedWriter& cache = writeStream.Init (kNeedsInstanceIDRemapping); + cache.Init (m_FileCacher, alignedByteStart, 0, false); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + cache.End (); + } +*/ + +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + string label = Format("--- !u!%d &%d\n", info.classID, (int)fileID); + WriteTextSerialized (label, object, mask); + } + else +#endif + if (!ShouldSwapEndian()) + { + // Set write marker to end of file and register the objects position in file + StreamedBinaryWrite<false> writeStream; + + CachedWriter& cache = writeStream.Init (*m_CachedWriter, mask, BuildTargetSelection(static_cast<BuildTargetPlatform>(m_TargetPlatform),m_SubTarget), buildUsage); + char kZeroAlignment[kFileAlignment] = {0, 0, 0, 0, 0, 0, 0, 0}; + cache.Write (kZeroAlignment, alignedByteStart - unalignedByteStart); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + + *m_CachedWriter = cache; + } + else + { + // Set write marker to end of file and register the objects position in file + StreamedBinaryWrite<true> writeStream; + CachedWriter& cache = writeStream.Init (*m_CachedWriter, mask, BuildTargetSelection(static_cast<BuildTargetPlatform>(m_TargetPlatform),m_SubTarget), buildUsage); + char kZeroAlignment[kFileAlignment] = {0, 0, 0, 0, 0, 0, 0, 0}; + cache.Write (kZeroAlignment, alignedByteStart - unalignedByteStart); + + // Write the object + object.VirtualRedirectTransfer (writeStream); + + *m_CachedWriter = cache; + } + + info.byteSize = m_CachedWriter->GetPosition() - info.byteStart; +} +#endif + +#if !UNITY_EXTERNAL_TOOL + +enum { kMonoBehaviourClassID = 114 }; +static void OutOfBoundsReadingError (int classID, int expected, int was) +{ + if (classID == kMonoBehaviourClassID) + { + // This code should not access any member variables + // because that might dereference pointers etc during threaded loading. + ErrorString(Format("A script behaviour has a different serialization layout when loading. (Read %d bytes but expected %d bytes)\nDid you #ifdef UNITY_EDITOR a section of your serialized properties in any of your scripts?", was, expected)); + } + else + { + ErrorString(Format("Mismatched serialization in the builtin class '%s'. (Read %d bytes but expected %d bytes)", Object::ClassIDToString(classID).c_str(), was, expected)); + } +} + +void SerializedFile::ReadObject (LocalIdentifierInFileType fileID, int instanceId, ObjectCreationMode mode, bool isPersistent, TypeTree** oldTypeTree, bool* didChangeTypeTree, Object** outObjectPtr) +{ +#if UNITY_WII + wii::ProcessShutdownAndReset(); + wii::ProcessLoadingScreen(); +#endif +// printf_console("Reading instance: %d fileID: %d filePtr: %d\n", instanceId, fileID, this); + + ObjectMap::iterator iter = m_Object.find (fileID); + + // Test if the object is in stream + if (iter == m_Object.end ()) + return; + + if (iter->second.isDestroyed) + { + return; + } + + const ObjectInfo& info = iter->second; + + // Create empty object + Object* objectPtr = *outObjectPtr; + if (objectPtr == NULL) + { + *outObjectPtr = objectPtr = Object::Produce (info.classID, instanceId, kMemBaseObject, mode); + } + + if (objectPtr == NULL) + { + ErrorString ("Could not produce class with ID " + IntToString (info.classID)); + return; + } + SET_ALLOC_OWNER(this); +#if UNITY_EDITOR + (**outObjectPtr).SetFileIDHint (fileID); +#endif + + #if SUPPORT_SERIALIZED_TYPETREES + AssertIf (objectPtr->GetClassID () != info.classID); + bool perClassTypeTree = true; + Type* type = NULL; + if (!m_Type.empty() +#if SUPPORT_TEXT_SERIALIZATION + && !m_IsTextFile +#endif + ) + { + // Find TypeTree + type = &m_Type[info.typeID]; + AssertIf (type->GetOldType () == NULL); + + perClassTypeTree = !objectPtr->GetNeedsPerObjectTypeTree (); + + AssertIf (perClassTypeTree && objectPtr->GetClassID () != info.typeID); + // If we have a per class type tree we do not generate a typetree before reading the object + // That also means we always use SafeBinaryRead. + + // Setup new header type data + // If we have a perObject TypeTree we dont need the new type since we safebinaryread anyway. + // If we have a per class typetree we generate the typetree only once since it can't change + // while the application is running + if (type->GetNewType () == NULL && perClassTypeTree) + { + // Create new type and init it using a proxy transfer + TypeTree* typeTree = UNITY_NEW (TypeTree, kMemTypeTree); + + int typeTreeOptions = kDontRequireAllMetaFlags | m_Options; + GenerateTypeTree (*objectPtr, typeTree, typeTreeOptions); + + // Register the type tree + type->SetNewType (typeTree); + + #if DEBUG_LOG_TYPETREE_MISMATCHES // Log when loaded typetree is out of sync. + if (!type->EqualTypes ()) + { + printf_console("Typetree mismatch in path: %s\n", m_DebugPath.c_str()); + printf_console("TYPETREE USED BY RUNTIME IS: \n"); + string buffer0; + type->GetOldType()->DebugPrint(buffer0); + printf_console(buffer0.c_str()); + + printf_console("TYPETREE IN FILE IS:\n"); + string buffer1; + type->GetNewType()->DebugPrint(buffer1); + printf_console(buffer1.c_str()); + } + #endif + } + AssertIf (type->EqualTypes () && !perClassTypeTree); + } + #endif + + int options = kNeedsInstanceIDRemapping | m_Options; + if (ShouldSwapEndian()) + options |= kSwapEndianess; + if (mode == kCreateObjectFromNonMainThread) + options |= kThreadedSerialization; + + objectPtr->SetIsPersistent(isPersistent); + + int byteStart = info.byteStart + m_ReadOffset; + + #if SUPPORT_SERIALIZED_TYPETREES + // Fill object with data + + // Never use the SafeBinaryRead code path for Meshes serialized with 3.5, because that needs a special workaround, + // which requires using StreamedBinaryRead. + if (type != NULL && !type->EqualTypes () && !((options & kWorkaround35MeshSerializationFuckup) && info.classID == ClassID(Mesh))) + { + SafeBinaryRead readStream; + + CachedReader& cache = readStream.Init (*type->GetOldType (), byteStart, info.byteSize , options); + cache.InitRead (*m_ReadFile, byteStart, info.byteSize); + Assert(m_ResourceImageGroup.resourceImages[0] == NULL); + + objectPtr->Reset (); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart > info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = perClassTypeTree; + } + else + #endif + { + // we will read up that object - no need to call Reset as we will construct it fully + objectPtr->HackSetResetWasCalled(); + + ///@TODO: Strip endianess! +#if SUPPORT_TEXT_SERIALIZATION + if (m_IsTextFile) + { + YAMLRead readStream (m_ReadFile, byteStart, byteStart + info.byteSize, options, &m_DebugPath, info.debugLineStart); + objectPtr->VirtualRedirectTransfer (readStream); + *didChangeTypeTree = false; + } + else +#endif + if (!ShouldSwapEndian()) + { + StreamedBinaryRead<false> readStream; + CachedReader& cache = readStream.Init (options); + cache.InitRead (*m_ReadFile, info.byteStart + m_ReadOffset, info.byteSize); + cache.InitResourceImages (m_ResourceImageGroup); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart != info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = false; + } + else + { + #if SUPPORT_SERIALIZED_TYPETREES + StreamedBinaryRead<true> readStream; + CachedReader& cache = readStream.Init (options); + + cache.InitRead (*m_ReadFile, byteStart, info.byteSize); + Assert(m_ResourceImageGroup.resourceImages[0] == NULL); + + // Read the object + objectPtr->VirtualRedirectTransfer (readStream); + int position = cache.End (); + if (position - byteStart != info.byteSize) + OutOfBoundsReadingError (info.classID, info.byteSize, position - byteStart); + + *didChangeTypeTree = false; + #else + AssertString("reading endian swapped is not supported"); + #endif + } + } + + #if SUPPORT_SERIALIZED_TYPETREES + if (type) + *oldTypeTree = type->GetOldType (); + else + *oldTypeTree = NULL; + #endif + + // Setup hide flags when loading from a resource file + if (m_Options & kIsBuiltinResourcesFile) + objectPtr->SetHideFlagsObjectOnly(Object::kHideAndDontSave | Object::kHideInspector); + + return; +} + +#endif + +bool SerializedFile::IsEmpty () const +{ + for (ObjectMap::const_iterator i = m_Object.begin (); + i != m_Object.end (); ++i) + { + if (!i->second.isDestroyed) + return false; + } + return true; +} + +void SerializedFile::AddExternalRef (const FileIdentifier& pathName) +{ + // Dont' check for pathname here - it can be empty, if we are getting a GUID from + // a text serialized file, and the file belonging to the GUID is missing. In that + // case we just keep the GUID. + #if SUPPORT_SERIALIZE_WRITE + Assert (m_CachedWriter != NULL); + #endif + m_Externals.push_back (pathName); + m_Externals.back().CheckValidity(); +} + +void InitializeStdConverters () +{ + RegisterAllowTypeNameConversion ("PPtr<TransformComponent>", "PPtr<Transform>"); + RegisterAllowTypeNameConversion ("PPtr<FileTexture>", "PPtr<Texture2D>"); + RegisterAllowTypeNameConversion ("PPtr<FileTexture>", "PPtr<Texture>"); + RegisterAllowTypeNameConversion ("PPtr<Animation>", "PPtr<AnimationClip>"); + RegisterAllowTypeNameConversion ("PPtr<GOComponent>", "PPtr<Component>"); + RegisterAllowTypeNameConversion ("PPtr<Script>", "PPtr<TextAsset>"); + RegisterAllowTypeNameConversion ("UniqueIdentifier", "GUID"); + RegisterAllowTypeNameConversion ("Vector3f", "Vector2f"); + RegisterAllowTypeNameConversion ("Vector3f", "Vector4f"); + RegisterAllowTypeNameConversion ("vector_set", "set"); + RegisterAllowTypeNameConversion ("vector_map", "map"); + RegisterAllowTypeNameConversion ("map", "vector"); + RegisterAllowTypeNameConversion ("set", "vector"); + RegisterAllowTypeNameConversion ("list", "vector"); + RegisterAllowTypeNameConversion ("deque", "vector"); + RegisterAllowTypeNameConversion ("dynamic_array", "vector"); + RegisterAllowTypeNameConversion ("TypelessData", "dynamic_array"); + RegisterAllowTypeNameConversion ("tricky_vector", "vector"); + RegisterAllowTypeNameConversion ("PPtr<DataTemplate>", "PPtr<Prefab>"); + RegisterAllowTypeNameConversion ("PPtr<EditorExtension>", "PPtr<Object>"); + + // Support for converting MonoBehaviour GUIStyle arrays to C++ native GUIStyle + RegisterAllowTypeNameConversion ("GUIStyle", "vector"); + RegisterAllowTypeNameConversion ("Generic Mono", "GUIStyle"); + RegisterAllowTypeNameConversion ("Generic Mono", "RectOffset"); + RegisterAllowTypeNameConversion ("Generic Mono", "GUIStyleState"); + RegisterAllowTypeNameConversion ("PPtr<$Texture2D>", "PPtr<Texture2D>"); + RegisterAllowTypeNameConversion ("PPtr<$Font>", "PPtr<Font>"); + + RegisterAllowTypeNameConversion ("PPtr<AvatarBodyMask>", "PPtr<AvatarMask>"); + RegisterAllowTypeNameConversion ("PPtr<AnimationSet>", "PPtr<AnimatorOverrideController>"); + RegisterAllowTypeNameConversion ("AvatarSkeletonMaskElement", "TransformMaskElement"); + RegisterAllowTypeNameConversion ("HumanLayerConstant", "LayerConstant"); + RegisterAllowTypeNameConversion ("PPtr<AnimatorController>", "PPtr<RuntimeAnimatorController>"); + +#if SUPPORT_SERIALIZED_TYPETREES + REGISTER_CONVERTER (float, double); + REGISTER_CONVERTER (double, float); + + REGISTER_CONVERTER (int, float); + + #define REGISTER_BASIC_INTEGERTYPE_CONVERTER(x) \ + REGISTER_CONVERTER (x, UInt64); \ + REGISTER_CONVERTER (x, SInt64); \ + REGISTER_CONVERTER (x, SInt32); \ + REGISTER_CONVERTER (x, UInt32); \ + REGISTER_CONVERTER (x, UInt16); \ + REGISTER_CONVERTER (x, SInt16); \ + REGISTER_CONVERTER (x, UInt8); \ + REGISTER_CONVERTER (x, SInt8); \ + REGISTER_CONVERTER (x, bool) + + + REGISTER_BASIC_INTEGERTYPE_CONVERTER(UInt64); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(SInt32); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(UInt32); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(UInt16); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(SInt16); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(UInt8); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(SInt8); + REGISTER_BASIC_INTEGERTYPE_CONVERTER(bool); + + #undef REGISTER_BASIC_INTEGERTYPE_CONVERTER + +#endif +} + +void CleanupStdConverters() +{ + ClearTypeNameConversion(); + SafeBinaryRead::CleanupConverterTable(); +} + +#if UNITY_EDITOR +bool SerializedFile::ExtractObjectData (LocalIdentifierInFileType fileID, SerializedFile::ObjectData& data) +{ + ObjectMap::iterator iter = m_Object.find (fileID); + if (iter == m_Object.end ()) + return false; + + const ObjectInfo& info = iter->second; + SET_ALLOC_OWNER(this); + // Find TypeTree + if (info.typeID) + { + Type& type = m_Type[info.typeID]; + AssertIf (type.GetOldType () == NULL); + data.typeTree = type.GetOldType (); + } + else + data.typeTree = NULL; + + data.classID = info.classID; + + if(info.byteSize > 0) + { + CachedReader cache; + cache.InitRead (*m_ReadFile, m_ReadOffset + info.byteStart, info.byteSize); + + data.data.resize_uninitialized(info.byteSize); + cache.Read(data.data.begin(), info.byteSize); + cache.End(); + } + return true; +} + +void FileIdentifier::CheckValidity () +{ + FatalErrorIf (type == kMetaAssetType && !pathName.empty()); + FatalErrorIf (type == kSerializedAssetType && !pathName.empty()); + FatalErrorIf (type == kDeprecatedCachedAssetType); +} + +void FileIdentifier::Fix_3_5_BackwardsCompatibility () +{ + // Backwards compatibility with 3.5 and before. + // We no longer store the path for asset types. (It is implicit by the GUID) + if (type == FileIdentifier::kDeprecatedCachedAssetType) + type = FileIdentifier::kMetaAssetType; + if (type == FileIdentifier::kSerializedAssetType || type == FileIdentifier::kMetaAssetType) + pathName.clear(); +} + +#endif diff --git a/Runtime/Serialize/SerializedFile.h b/Runtime/Serialize/SerializedFile.h new file mode 100644 index 0000000..c3e431a --- /dev/null +++ b/Runtime/Serialize/SerializedFile.h @@ -0,0 +1,264 @@ +#ifndef SERIALIZEDFILE_H +#define SERIALIZEDFILE_H + +#include <map> +#include <string> +#include "TypeTree.h" +#include "Runtime/Serialize/TransferFunctions/SerializeTransfer.h" +#include "FileCache.h" +#include "Runtime/BaseClasses/BaseObject.h" +#include "Runtime/Utilities/vector_map.h" +#include "Runtime/Utilities/dynamic_block_vector.h" +#if !GAMERELEASE +#include "Runtime/Utilities/GUID.h" +#endif + +extern const char* kUnityTextMagicString; + +using std::map; +using std::string; +using std::list; +using std::vector; + +class CachedWriter; +enum ObjectCreationMode; +struct ResourceImageGroup; + +struct FileIdentifier +{ + enum { kNonAssetType = 0, kDeprecatedCachedAssetType = 1, kSerializedAssetType = 2, kMetaAssetType = 3 }; + + UnityStr pathName; + SInt32 type; + + #if GAMERELEASE + struct GUIDPlaceHolder { UInt32 data[4]; }; + GUIDPlaceHolder guid; + + void CheckValidity () {} + + #else + + UnityGUID guid; + + FileIdentifier (const string& p, const UnityGUID& g, int t) + : pathName (p), guid (g), type (t) + { + CheckValidity (); + } + + void CheckValidity (); + void Fix_3_5_BackwardsCompatibility (); + + bool operator < (const FileIdentifier &other) const + { + if (guid < other.guid) + return true; + else if (guid != other.guid) + return false; + else return type < other.type; + } + #endif + + FileIdentifier () { type = 0; } + + #if SUPPORT_TEXT_SERIALIZATION + DECLARE_SERIALIZE (FileIdentifier); + #endif +}; + +class SerializedFile +{ + struct ObjectInfo + { + SInt32 byteStart; + SInt32 byteSize; + SInt32 typeID; + SInt16 classID; + UInt16 isDestroyed; +#if SUPPORT_TEXT_SERIALIZATION + SInt32 debugLineStart; +#endif + }; + + typedef vector_map<LocalIdentifierInFileType, ObjectInfo> ObjectMap; + + +#if SUPPORT_TEXT_SERIALIZATION + bool m_IsTextFile; +#endif + + unsigned m_ReadOffset; + + #if SUPPORT_SERIALIZED_TYPETREES + typedef map<SInt32, Type> TypeMap; + TypeMap m_Type; + #endif + ObjectMap m_Object; + UInt8 m_FileEndianess; + + bool m_IsDirty; + bool m_MemoryStream; + bool m_CachedFileStream; + bool m_HasErrors; + UInt32 m_Options; + UInt32 m_TargetPlatform; // enum BuildTargetPlatform + int m_SubTarget; + UInt32 m_WriteDataOffset; + + dynamic_block_vector<FileIdentifier>m_Externals; + + CacheReaderBase* m_ReadFile; + ResourceImageGroup m_ResourceImageGroup; + + #if ENABLE_PROFILER || UNITY_EDITOR + std::string m_DebugPath; + #endif + + #if SUPPORT_SERIALIZE_WRITE + CachedWriter* m_CachedWriter; + + //unsigned m_ObjectBufferStart; + #endif + public: + + enum { kSectionAlignment = 16 }; + + enum + { + kLittleEndian = 0, + kBigEndian = 1, + + #if UNITY_BIG_ENDIAN + kActiveEndianess = kBigEndian, + kOppositeEndianess = kLittleEndian + #else + kActiveEndianess = kLittleEndian, + kOppositeEndianess = kBigEndian + #endif + }; + + SerializedFile (); + + // options: kSerializeGameRelease, kSwapEndianess, kBuildPlayerOnlySerializeBuildProperties + bool InitializeWrite (CachedWriter& cachedWriter, BuildTargetSelection target, int options); + bool InitializeRead (const std::string& path, ResourceImageGroup& resourceImage, unsigned cacheSize, unsigned cacheCount, int options, int readOffset = 0); + bool InitializeMemory (const string& path, UInt8* buffer, unsigned size, int options); + bool InitializeMemoryBlocks (const string& path, UInt8** buffer, unsigned size, unsigned offset, int options); + + ~SerializedFile (); + + static void DeleteNoFlush (SerializedFile* file); + + #if ENABLE_PROFILER || UNITY_EDITOR + void SetDebugPath(const std::string& path) { m_DebugPath = path; } + const std::string& GetDebugPath() const { return m_DebugPath; } + #endif + + // Writes an object with id to the file. (Possibly overriding any older versions) + // Writing to a stream which includes objects with an older typetree version is not possible and false will be returned + void WriteObject (Object& object, LocalIdentifierInFileType fileID, const BuildUsageTag& buildUsage); + + // Reads the object referenced by id from disk + // Returns a pointer to the object. (NULL if no object was found on disk) + // object is either PRODUCED or the object already in memory referenced by id is used + // isMarkedDestroyed is a returned by value (non-NULL) + // registerInstanceID should the instanceID be register with the ID To Object lookup (false for threaded loading) + // And reports whether the object read was marked as destroyed or not + void ReadObject (LocalIdentifierInFileType fileID, int instanceId, ObjectCreationMode mode, bool isPersistent, TypeTree** oldTypeTree, bool* didChangeTypeTree, Object** readObject); + + // Returns whether or not the object referenced by id was destroyed + bool DestroyObject (LocalIdentifierInFileType id); + + // objects: On return, all fileIDs to all objects in this Serialze + void GetAllFileIDs (vector<LocalIdentifierInFileType>* objects)const; + void GetAllFileIDsUnchecked (vector<LocalIdentifierInFileType>* objects)const; + + // Returns the biggest id of all the objects in the file. + // if no objects are in the file 0 is returned + LocalIdentifierInFileType GetHighestID () const; + + // Returns whether or not an object is available in the stream + bool IsAvailable (LocalIdentifierInFileType id) const; + // Returns the classID of the object at id + int GetClassID (LocalIdentifierInFileType id) const; + + // Returns the size the object takes up on the disk + int GetByteSize (LocalIdentifierInFileType id) const; + + // Returns the seek position in the file where the object with id is stored + int GetByteStart (LocalIdentifierInFileType id) const; + + // Returns the seek position in the file where the object with id is stored + const TypeTree* GetTypeTree (LocalIdentifierInFileType id); + + // Are there any objects stored in this serialize. + bool IsEmpty () const; + + bool HasErrors () const { return m_HasErrors; } + + #if UNITY_EDITOR + struct ObjectData + { + int classID; + dynamic_array<UInt8> data; + TypeTree* typeTree; + }; + bool ExtractObjectData (LocalIdentifierInFileType fileID, ObjectData& data); + #endif + + // Get/Set the list of FileIdentifiers this file uses + const dynamic_block_vector<FileIdentifier>& GetExternalRefs ()const { return m_Externals; } + + // Add an external reference + void AddExternalRef (const FileIdentifier& pathName); + + // Is the header of the file written? + bool IsFileDirty () const { return m_IsDirty; } + + inline bool ShouldSwapEndian () const { return m_FileEndianess != kActiveEndianess; } + + bool IsMemoryStream () const { return m_MemoryStream; } + bool IsCachedFileStream () const { return m_CachedFileStream; } + + void SetIsCachedFileStream (bool cache) { m_CachedFileStream = cache; } + + #if SUPPORT_SERIALIZE_WRITE + bool FinishWriting (); + #endif + + #if SUPPORT_TEXT_SERIALIZATION + bool IsTextFile () const { return m_IsTextFile; } + #endif +private: + // Writes everything that is in the caches out to the disk. + void Flush (); + + bool FinalizeInit (int options); + bool ReadHeader (); + + template<bool kSwap> bool ReadMetadata (int version, unsigned dataOffset, UInt8 const* data, size_t length, size_t dataFileSize); + +#if SUPPORT_SERIALIZE_WRITE + template<bool kSwap> void BuildMetadataSection (SerializationCache& cache, unsigned dataOffsetInFile); + template<bool kSwap> bool WriteHeader (SerializationCache& cache); +#endif + +#if SUPPORT_TEXT_SERIALIZATION + template<class T> + void WriteTextSerialized (string& label, T &data, int options); + bool IndexTextFile (); + bool ReadHeaderText (); +#endif + + +}; + +#if SUPPORT_TEXT_SERIALIZATION +bool IsSerializedFileTextFile(string pathName); +#endif + +void InitializeStdConverters (); +void CleanupStdConverters(); + +#endif diff --git a/Runtime/Serialize/SerializedFileTests.cpp b/Runtime/Serialize/SerializedFileTests.cpp new file mode 100644 index 0000000..7847aa9 --- /dev/null +++ b/Runtime/Serialize/SerializedFileTests.cpp @@ -0,0 +1,39 @@ +#include "UnityPrefix.h" + +#if ENABLE_UNIT_TESTS + +#include "SerializedFile.h" +#include "External/UnitTest++/src/UnitTest++.h" +#include "Runtime/Serialize/FileCache.h" + +SUITE (SerializedFile) +{ + #if SUPPORT_SERIALIZE_WRITE + TEST_FIXTURE (SerializedFile, ReadWriteSerializedFileWorks) + { + CachedWriter writer; + FileCacherWrite writeFile; + writeFile.InitWriteFile("test.serialized", 16); + writer.InitWrite(writeFile); + + SerializedFile* file = UNITY_NEW_AS_ROOT (SerializedFile(), kMemSerialization, "SerializedFile", ""); + CHECK(file->InitializeWrite (writer, BuildTargetSelection::NoTarget(), 0)); + CHECK(file->FinishWriting()); + UNITY_DELETE(file, kMemSerialization); + + file = UNITY_NEW_AS_ROOT (SerializedFile(), kMemSerialization, "SerializedFile", ""); + + ResourceImageGroup resources; + CHECK(file->InitializeRead("test.serialized", resources, 16, 2, 0)); + CHECK(!file->IsFileDirty()); + CHECK(file->IsEmpty()); + + UNITY_DELETE(file, kMemSerialization); + DeleteFile("test.serialized"); + } + #endif + +} // SUITE + + +#endif diff --git a/Runtime/Serialize/SwapEndianArray.h b/Runtime/Serialize/SwapEndianArray.h new file mode 100644 index 0000000..2fbf30f --- /dev/null +++ b/Runtime/Serialize/SwapEndianArray.h @@ -0,0 +1,27 @@ +#ifndef SWAPENDIANARRAY_H +#define SWAPENDIANARRAY_H + +#include "SwapEndianBytes.h" + +inline void SwapEndianArray (void* data, int bytesPerComponent, int count) +{ + + if (bytesPerComponent == 2) + { + UInt16* p = (UInt16*)data; + for (int i=0;i<count;i++) + SwapEndianBytes (*p++); + } + else if (bytesPerComponent == 4) + { + UInt32* p = (UInt32*)data; + for (int i=0;i<count;i++) + SwapEndianBytes (*p++); + } + else + { + AssertIf (bytesPerComponent != 1); + } +} + +#endif diff --git a/Runtime/Serialize/SwapEndianBytes.h b/Runtime/Serialize/SwapEndianBytes.h new file mode 100644 index 0000000..2a55246 --- /dev/null +++ b/Runtime/Serialize/SwapEndianBytes.h @@ -0,0 +1,89 @@ +#ifndef SWAPENDIANBYTES_H +#define SWAPENDIANBYTES_H + +inline void SwapEndianBytes (char&) { } +inline void SwapEndianBytes (unsigned char&) { } +inline void SwapEndianBytes (bool&) { } +inline void SwapEndianBytes (signed char&) { } + +#if UNITY_WII +inline void SwapEndianBytes (register UInt16& i) +{ + __asm { + lhbrx r3, 0, i + sth r3, 0(i) + } +} +#else +inline void SwapEndianBytes (UInt16& i) { i = static_cast<UInt16>((i << 8) | (i >> 8)); } +#endif + +inline void SwapEndianBytes (SInt16& i) { SwapEndianBytes (reinterpret_cast<UInt16&> (i)); } + +#if UNITY_WII +inline void SwapEndianBytes (register UInt32& i) +{ + __asm { + lwbrx r3, 0, i + stw r3, 0(i) + } +} +#else +inline void SwapEndianBytes (UInt32& i) { i = (i >> 24) | ((i >> 8) & 0x0000ff00) | ((i << 8) & 0x00ff0000) | (i << 24); } +#endif + +inline void SwapEndianBytes (SInt32& i) { SwapEndianBytes (reinterpret_cast<UInt32&> (i)); } +inline void SwapEndianBytes (float& i) { SwapEndianBytes (reinterpret_cast<UInt32&> (i)); } + +inline void SwapEndianBytes (UInt64& i) +{ +#if UNITY_WII + register unsigned int temp1, temp2; + register UInt32* p0 = reinterpret_cast<UInt32*>( &i ); + register UInt32* p1 = reinterpret_cast<UInt32*>( &i ) + 1; + __asm { + lwbrx temp1, 0, p0 + lwbrx temp2, 0, p1 + stw temp1, 0(p1) + stw temp2, 0(p0) + } +#else + UInt32* p = reinterpret_cast<UInt32*>(&i); + UInt32 u = (p[0] >> 24) | (p[0] << 24) | ((p[0] & 0x00ff0000) >> 8) | ((p[0] & 0x0000ff00) << 8); + p[0] = (p[1] >> 24) | (p[1] << 24) | ((p[1] & 0x00ff0000) >> 8) | ((p[1] & 0x0000ff00) << 8); + p[1] = u; +#endif +} + +inline void SwapEndianBytes (SInt64& i) { SwapEndianBytes (reinterpret_cast<UInt64&> (i)); } +inline void SwapEndianBytes (double& i) { SwapEndianBytes (reinterpret_cast<UInt64&> (i)); } + +#if UNITY_64 && UNITY_OSX +inline void SwapEndianBytes (size_t &i) { SwapEndianBytes (reinterpret_cast<UInt64&> (i)); } +#endif + +inline bool IsBigEndian () +{ + #if UNITY_BIG_ENDIAN + return true; + #else + return false; + #endif +} + +inline bool IsLittleEndian () +{ + return !IsBigEndian(); +} + +#if UNITY_BIG_ENDIAN +#define SwapEndianBytesBigToNative(x) +#define SwapEndianBytesLittleToNative(x) SwapEndianBytes(x) +#define SwapEndianBytesNativeToLittle(x) SwapEndianBytes(x) +#elif UNITY_LITTLE_ENDIAN +#define SwapEndianBytesBigToNative(x) SwapEndianBytes(x) +#define SwapEndianBytesLittleToNative(x) +#define SwapEndianBytesNativeToLittle(x) +#endif + +#endif diff --git a/Runtime/Serialize/TransferFunctionFwd.h b/Runtime/Serialize/TransferFunctionFwd.h new file mode 100644 index 0000000..2307dcf --- /dev/null +++ b/Runtime/Serialize/TransferFunctionFwd.h @@ -0,0 +1,14 @@ +#ifndef TRANFER_FUNCTION_FWD_H_ +#define TRANFER_FUNCTION_FWD_H_ + +class ProxyTransfer; +class RemapPPtrTransfer; +class SafeBinaryRead; +template<bool kSwapEndianess> class StreamedBinaryRead; +template<bool kSwapEndianess> class StreamedBinaryWrite; +class YAMLRead; +class YAMLWrite; +class BlobWrite; +class BlobSize; + +#endif diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp b/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp new file mode 100644 index 0000000..de4f731 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp @@ -0,0 +1,250 @@ +#include "UnityPrefix.h" +#include "ProxyTransfer.h" +#include "Runtime/Utilities/Word.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + +using namespace std; + +ProxyTransfer::ProxyTransfer (TypeTree& t, int options, void* objectPtr, int objectSize) + : m_TypeTree (t), + m_ActiveFather (NULL) +{ + m_Flags = options; + + m_ObjectPtr = (char*)objectPtr; + m_ObjectSize = objectSize; + m_Index = 0; + m_SimulatedByteOffset = 0; + m_RequireTypelessData = false; + m_DidErrorAlignment = false; +} + +void ProxyTransfer::BeginArrayTransfer (const char* name, const char* typeString, SInt32& size, TransferMetaFlags metaFlag) +{ + AssertIf (m_ActiveFather->m_IsArray); + BeginTransfer (name, typeString, NULL, metaFlag); + m_ActiveFather->m_IsArray = true; + + // transfer size + Transfer (size, "size"); +} + +void ProxyTransfer::BeginTransfer (const char* name, const char* typeString, char* dataPtr, TransferMetaFlags metaFlag) +{ + #if !UNITY_RELEASE + if (m_RequireTypelessData) + { + AssertString ("TransferTypeless needs to be followed by TransferTypelessData with no other variables in between!"); + } + #endif + + #if DEBUGMODE + // skip this check for debug mode inspector, as we can have interface names from C# in the debug data. + if (!(metaFlag & kDebugPropertyMask) && (strstr (name, ".") != NULL || strstr (name, "Array[") != NULL)) + { + string s = "Illegal serialize property name :"; + GetTypePath (m_ActiveFather, s); + s += name; + s += "\n The name may not contain '.' or Array["; + ErrorString (s); + } + #endif + + TypeTree* typeTree; + // Setup a normal typetree child + if (m_ActiveFather != NULL) + { + // Check for multiple occurences of same name + #if DEBUGMODE + TypeTree::TypeTreeList::iterator i; + for (i=m_ActiveFather->m_Children.begin ();i != m_ActiveFather->m_Children.end ();++i) + { + if (i->m_Name == name) + { + string s = "The same field name is serialized multiple names in the class or it's parent class. This is not supported: "; + GetTypePath (m_ActiveFather, s); + s += name; + ErrorString (s); + } + } + #endif + + m_ActiveFather->m_Children.push_back (TypeTree ()); + // Setup new type + typeTree = &m_ActiveFather->m_Children.back (); + typeTree->m_Father = m_ActiveFather; + typeTree->m_Type = typeString; + typeTree->m_Name = name; + typeTree->m_MetaFlag = metaFlag | m_ActiveFather->m_MetaFlag; + AssertIf(typeTree->m_MetaFlag & kAlignBytesFlag); + typeTree->m_MetaFlag &= ~(kAnyChildUsesAlignBytesFlag); + typeTree->m_ByteSize = 0; + } + // Setup root TypeTree + else + { + m_TypeTree.m_Father = NULL; + m_TypeTree.m_Type = typeString; + m_TypeTree.m_Name = name; + m_TypeTree.m_MetaFlag = metaFlag; + m_TypeTree.m_ByteSize = 0; + typeTree = &m_TypeTree; + } + + // Calculate typetree index + if ((typeTree->m_MetaFlag & kDebugPropertyMask) == 0) + typeTree->m_Index = m_Index++; + else + { + if (m_Flags & kIgnoreDebugPropertiesForIndex) + typeTree->m_Index = -1; + else + typeTree->m_Index = m_Index++; + } + + m_ActiveFather = typeTree; + + int offset = dataPtr - m_ObjectPtr; + if (m_ObjectPtr && dataPtr && offset >= 0 && offset < m_ObjectSize) + m_ActiveFather->m_ByteOffset = offset; + + m_ActiveFather->m_DirectPtr = dataPtr; +} + +void ProxyTransfer::AssertContainsNoPPtr (const TypeTree* typeTree) +{ + AssertIf(typeTree->m_Type.find("PPtr<") == 0); + for (TypeTree::const_iterator i=typeTree->begin();i != typeTree->end();i++) + AssertContainsNoPPtr(&*i); +} + +void ProxyTransfer::AssertOptimizeTransfer (int sizeofSize) +{ + if (m_ActiveFather->IsBasicDataType()) + { + AssertIf(sizeofSize != m_ActiveFather->m_ByteSize); + return; + } + + int size = 0; + int baseOffset = m_ActiveFather->m_ByteOffset; + for (TypeTree::const_iterator i=m_ActiveFather->begin();i != m_ActiveFather->end();i++) + { + AssertOptimizeTransferImpl(*i, baseOffset, &size); + } + + // Assert if serialized size is different from sizeof size. + // - Ignore when serializing for game release. We might be serializing differently in that case. (AnimationCurves) + AssertIf(sizeofSize != size && (m_Flags & kSerializeGameRelease) == 0); +} + +void ProxyTransfer::AssertOptimizeTransferImpl (const TypeTree& typetree, int baseOffset, int* totalSize) +{ + if (typetree.m_ByteOffset != -1) + AssertIf (typetree.m_ByteOffset - baseOffset != *totalSize); + + AssertIf (typetree.m_MetaFlag & kAlignBytesFlag); + + if (typetree.IsBasicDataType()) + { + *totalSize += typetree.m_ByteSize; + return; + } + + for (TypeTree::const_iterator i=typetree.begin();i != typetree.end();i++) + AssertOptimizeTransferImpl(*i, baseOffset, totalSize); +} + +void ProxyTransfer::EndTransfer () +{ + TypeTree* current = m_ActiveFather; + // Add bytesize to parent! + m_ActiveFather = m_ActiveFather->m_Father; + if (m_ActiveFather) + { + if (current->m_ByteSize != -1 && m_ActiveFather->m_ByteSize != -1) + m_ActiveFather->m_ByteSize += current->m_ByteSize; + else + m_ActiveFather->m_ByteSize = -1; + + // Propagate if any child uses alignment up to parents + if (current->m_MetaFlag & kAnyChildUsesAlignBytesFlag) + { + m_ActiveFather->m_MetaFlag |= kAnyChildUsesAlignBytesFlag; + } + + DebugAssertIf (m_ActiveFather->m_ByteSize == 0 && m_ActiveFather->m_Type == "Generic Mono"); + } +} + +void ProxyTransfer::EndArrayTransfer () +{ + m_ActiveFather->m_ByteSize = -1; + EndTransfer (); +} + +void ProxyTransfer::SetVersion (int version) +{ + // You can not set the version twice on the same type. + // Probably an inherited class already calls SetVersion + AssertIf (m_ActiveFather->m_Version != 1); + + m_ActiveFather->m_Version = version; +} + +void ProxyTransfer::TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer (name, "TypelessData", size, metaFlag); + + UInt8 temp; + Transfer (temp, "data", metaFlag); + + m_RequireTypelessData = true; + + EndArrayTransfer (); + + Align (); +} + +void ProxyTransfer::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + m_RequireTypelessData = false; +} + +void ProxyTransfer::Align () +{ + m_SimulatedByteOffset = Align4(m_SimulatedByteOffset); + + if (m_ActiveFather && !m_ActiveFather->m_Children.empty()) + { + m_ActiveFather->m_Children.back().m_MetaFlag |= kAlignBytesFlag; + m_ActiveFather->m_MetaFlag |= kAnyChildUsesAlignBytesFlag; + } + else + { + AssertString("Trying to align type data before anything has been serialized!"); + } +} + +#if UNITY_EDITOR +void ProxyTransfer::LogUnalignedTransfer () +{ + if (m_DidErrorAlignment) + return; + + // For now we only support 4 byte alignment + int size = m_ActiveFather->m_ByteSize; + if (size == 8) + size = 4; + if (m_SimulatedByteOffset % size == 0) + return; + + m_DidErrorAlignment = true; + + string path; + GetTypePath(m_ActiveFather, path); + LogString(Format("Unaligned transfer in '%s' at variable '%s'.\nNext unaligned data path: %s", m_TypeTree.m_Type.c_str(), m_ActiveFather->m_Name.c_str(), path.c_str())); +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransfer.h b/Runtime/Serialize/TransferFunctions/ProxyTransfer.h new file mode 100644 index 0000000..662ad05 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransfer.h @@ -0,0 +1,159 @@ +#ifndef PROXYTRANSFER_H +#define PROXYTRANSFER_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" + +class YAMLNode; +struct StreamingInfo; +struct ReduceCopyData; + +class EXPORT_COREMODULE ProxyTransfer : public TransferBase +{ + TypeTree& m_TypeTree; + TypeTree* m_ActiveFather; + char* m_ObjectPtr; + int m_ObjectSize; + int m_Index; + int m_SimulatedByteOffset; + bool m_DidErrorAlignment; + bool m_RequireTypelessData; + friend class MonoBehaviour; + +public: + + ProxyTransfer (TypeTree& t, int flags, void* objectPtr, int objectSize); + + void SetVersion (int version); + bool IsVersionSmallerOrEqual (int version) { Assert(m_ActiveFather->m_Version > version); return false; } + bool IsOldVersion (int version) { Assert(m_ActiveFather->m_Version > version); return false; } + bool NeedNonCriticalMetaFlags () { return (m_Flags & kDontRequireAllMetaFlags) == 0; } + + void AddMetaFlag (TransferMetaFlags flag) { m_ActiveFather->m_MetaFlag = m_ActiveFather->m_MetaFlag | flag; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + void TransferTypelessData (unsigned, void*, int metaData = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> inline + void TransferSTLStyleArrayWithElement (T& elementType, TransferMetaFlags metaFlag); + + void Align (); + + void BeginTransfer (const char* name, const char* typeString, char* data, TransferMetaFlags metaFlag); + void EndTransfer (); + + bool GetTransferFileInfo(unsigned* /*position*/, const char** /*filePath*/) const { return false; } + +private: + + void LogUnalignedTransfer (); + void AssertContainsNoPPtr (const TypeTree* typeTree); + void AssertOptimizeTransfer (int sizeofSize); + void AssertOptimizeTransferImpl (const TypeTree& typetree, int baseOffset, int* totalSize); + void CheckAlignment (); + + + void BeginArrayTransfer (const char* name, const char* typeString, SInt32& size, TransferMetaFlags metaFlag); + void EndArrayTransfer (); +}; + + +template<class T> inline +void ProxyTransfer::TransferSTLStyleArray (T& /*data*/, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + typename T::value_type p; + Transfer (p, "data"); + + // Make sure MightContainPPtr and AllowTransferOptimization is setup correctly + DebugAssertIf(SerializeTraits<T>::AllowTransferOptimization()); + + EndArrayTransfer (); +} + +template<class T> inline +void ProxyTransfer::TransferSTLStyleArrayWithElement (T& elementType, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + Transfer (elementType, "data"); + + EndArrayTransfer (); +} + + +template<class T> inline +void ProxyTransfer::TransferSTLStyleMap (T&, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + typename NonConstContainerValueType<T>::value_type p; + Transfer (p, "data"); + + #if !UNITY_RELEASE + DebugAssertIf(SerializeTraits<T>::AllowTransferOptimization()); + #endif + + EndArrayTransfer (); +} + +template<class T> inline +void ProxyTransfer::Transfer (T& data, const char* name, TransferMetaFlags metaFlag) +{ + BeginTransfer (name, SerializeTraits<T>::GetTypeString (&data), (char*)&data, metaFlag); + SerializeTraits<T>::Transfer (data, *this); + + // Make sure MightContainPPtr and AllowTransferOptimization is setup correctly + #if !UNITY_RELEASE + if (!SerializeTraits<T>::MightContainPPtr()) + AssertContainsNoPPtr(m_ActiveFather); + if (SerializeTraits<T>::AllowTransferOptimization()) + AssertOptimizeTransfer(SerializeTraits<T>::GetByteSize()); + #endif + + EndTransfer (); +} + +template<class T> inline +void ProxyTransfer::TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag) +{ + BeginTransfer (name, typeName, (char*)&data, metaFlag); + SerializeTraits<T>::Transfer (data, *this); + EndTransfer (); +} + + +template<class T> +inline void ProxyTransfer::TransferBasicData (T&) +{ + m_ActiveFather->m_ByteSize = SerializeTraits<T>::GetByteSize (); + #if UNITY_EDITOR + if (m_SimulatedByteOffset % m_ActiveFather->m_ByteSize != 0) + { + LogUnalignedTransfer(); + } + m_SimulatedByteOffset += m_ActiveFather->m_ByteSize; + #endif +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp b/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp new file mode 100644 index 0000000..2f62542 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp @@ -0,0 +1,216 @@ +#include "UnityPrefix.h" + +#if ENABLE_UNIT_TESTS +#include "External/UnitTest++/src/UnitTest++.h" +#include "Runtime/Serialize/TypeTree.h" +#include "Runtime/Serialize/TransferFunctions/ProxyTransfer.h" + +struct TypeTreeTestItem +{ +public: + DECLARE_SERIALIZE (TypeTreeTestItem); + + float m_float; + double m_double; + int m_int; // Same as SInt32 to serialization system. + unsigned int m_unsignedint; // Same as UInt32 to serialization system. + SInt64 m_SInt64; + UInt64 m_UInt64; + SInt32 m_SInt32; + UInt32 m_UInt32; + SInt16 m_SInt16; + UInt16 m_UInt16; + SInt8 m_SInt8; + UInt8 m_UInt8; + char m_char; + bool m_bool; + UnityStr m_UnityStr; +}; + +template<class TransferFunction> +void TypeTreeTestItem::Transfer (TransferFunction& transfer) +{ + TRANSFER (m_float); + TRANSFER (m_double); + TRANSFER (m_int); + TRANSFER (m_unsignedint); + TRANSFER (m_SInt64); + TRANSFER (m_UInt64); + TRANSFER (m_SInt32); + TRANSFER (m_UInt32); + TRANSFER (m_SInt16); + TRANSFER (m_UInt16); + TRANSFER (m_SInt8); + TRANSFER (m_UInt8); + TRANSFER (m_char); + TRANSFER (m_bool); + TRANSFER (m_UnityStr); +} + +SUITE (ProxyTransferTests) +{ + TEST (TypeTree_GenerateBaseTypes) + { + // Create test item. + TypeTreeTestItem item; + + // Populate test item. + item.m_float = 123.0f; + item.m_double = 456.0; + item.m_int = 2^31-1; + item.m_unsignedint = 2^32-1; + item.m_SInt64 = 2^63-1; + item.m_UInt64 = 2^64-1; + item.m_SInt32 = 2^31-1; + item.m_UInt32 = 2^32-1; + item.m_SInt16 = 2^15-1; + item.m_UInt16 = 2^16-1; + item.m_SInt8 = 2^7-1; + item.m_UInt8 = 2^8-1; + item.m_char = '@'; + item.m_bool = true; + item.m_UnityStr = "ProxyTransferTests"; + + // Generate type tree. + TypeTree tree; + ProxyTransfer proxy (tree, 0, &item, sizeof(TypeTreeTestItem)); + proxy.Transfer( item, "Base" ); + + // Validate the following expected output: + // + // TypeTree: Base Type:TypeTreeTestItem ByteSize:-1 MetaFlag:32768 + // m_float Type:float ByteSize:4 MetaFlag:0 + // m_double Type:double ByteSize:8 MetaFlag:0 + // m_int Type:int ByteSize:4 MetaFlag:0 + // m_unsignedint Type:unsigned int ByteSize:4 MetaFlag:0 + // m_SInt64 Type:SInt64 ByteSize:8 MetaFlag:0 + // m_UInt64 Type:UInt64 ByteSize:8 MetaFlag:0 + // m_SInt32 Type:SInt32 ByteSize:4 MetaFlag:0 + // m_UInt32 Type:UInt32 ByteSize:4 MetaFlag:0 + // m_SInt16 Type:SInt16 ByteSize:2 MetaFlag:0 + // m_UInt16 Type:UInt16 ByteSize:2 MetaFlag:0 + // m_SInt8 Type:SInt8 ByteSize:1 MetaFlag:0 + // m_UInt8 Type:UInt8 ByteSize:1 MetaFlag:0 + // m_char Type:char ByteSize:1 MetaFlag:0 + // m_bool Type:bool ByteSize:1 MetaFlag:0 + // m_UnityStr Type:string ByteSize:-1 MetaFlag:32768 + // Array Type:Array ByteSize:-1 MetaFlag:16385 IsArray + // size Type:SInt32 ByteSize:4 MetaFlag:1 + // data Type:char ByteSize:1 MetaFlag:1 + + // Check correct number of children. + const int expectedChildrenCount = 15; + CHECK_EQUAL (expectedChildrenCount, tree.m_Children.size()); + + // Transfer children for explicit checking. + TypeTree childTree[expectedChildrenCount]; + int childIndex = 0; + for( TypeTree::iterator treeItr = tree.begin(); treeItr != tree.end(); ++treeItr ) + childTree[childIndex++] = *treeItr; + + // Check children types... + + TypeTree* child = childTree; + CHECK_EQUAL (SerializeTraits<float>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_float", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<double>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_double", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_int", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_unsignedint", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt64>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt64", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt64>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt64", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt32", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt32>::GetTypeString(), child->m_Type); + CHECK_EQUAL ("m_UInt32", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt16>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt16", child->m_Name); + CHECK_EQUAL (2, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt16>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt16", child->m_Name); + CHECK_EQUAL (2, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt8>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt8", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt8>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt8", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<char>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_char", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL ("m_bool", child->m_Name); + CHECK_EQUAL (SerializeTraits<bool>::GetTypeString (), child->m_Type); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL ("m_UnityStr", child->m_Name); // String. + CHECK_EQUAL (SerializeTraits<UnityStr>::GetTypeString (), child->m_Type); + CHECK_EQUAL (-1, child->m_ByteSize); + CHECK_EQUAL (1, child->m_Children.size()); + CHECK_EQUAL ("Array", (*child->begin()).m_Name); // String is an array. + CHECK_EQUAL ("Array", (*child->begin()).m_Type); + CHECK_EQUAL (-1, (*child->begin()).m_ByteSize); + CHECK ( (*child->begin()).m_IsArray != 0); + CHECK_EQUAL ("size", (*(*child->begin()).begin()).m_Name); // Array size. + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), (*(*child->begin()).begin()).m_Type); + CHECK_EQUAL (4, (*(*child->begin()).begin()).m_ByteSize); + CHECK_EQUAL ("data", (*++(*child->begin()).begin()).m_Name); // Array data. + CHECK_EQUAL (SerializeTraits<char>::GetTypeString (), (*++(*child->begin()).begin()).m_Type); + CHECK_EQUAL (1, (*++(*child->begin()).begin()).m_ByteSize); + } +} + +#endif //ENABLE_UNIT_TESTS + diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp new file mode 100644 index 0000000..62c5a36 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" +#include "RemapPPtrTransfer.h" + +RemapPPtrTransfer::RemapPPtrTransfer (int flags, bool readPPtrs) +{ + m_ReadPPtrs = readPPtrs; + m_Flags = flags; + m_UserData = NULL; + m_GenerateIDFunctor = NULL; + m_MetaMaskStack.reserve(4); + m_MetaMaskStack.push_back (kNoTransferFlags); + m_CachedMetaMaskStackTop = kNoTransferFlags; +} + +void RemapPPtrTransfer::PushMetaFlag (TransferMetaFlags flag) +{ + m_MetaMaskStack.push_back (m_MetaMaskStack.back() | flag); + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +} + +void RemapPPtrTransfer::PopMetaFlag () +{ + m_MetaMaskStack.pop_back(); + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +} + +void RemapPPtrTransfer::AddMetaFlag (TransferMetaFlags flag) +{ + m_MetaMaskStack.back () |= flag; + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +}
\ No newline at end of file diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h new file mode 100644 index 0000000..853d83b --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h @@ -0,0 +1,178 @@ +#ifndef REMAPPPTRTRANSFER_H +#define REMAPPPTRTRANSFER_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" +#include "Runtime/Allocator/MemoryMacros.h" + +#include <stack> + +template<class T> +class PPtr; +template<class T> +class ImmediatePtr; +struct StreamingInfo; + +class GenerateIDFunctor +{ + public: + + /// Calls GenerateInstanceID for every PPtr that is found while transferring. + /// oldInstanceID is the instanceID of the PPtr. metaFlag is the ored metaFlag of the the Transfer trace to the currently transferred pptr. + /// After GenerateInstanceID returns, the PPtr will be set to the returned instanceID + /// If you dont want to change the PPtr return oldInstanceID + virtual SInt32 GenerateInstanceID (SInt32 oldInstanceID, TransferMetaFlags metaFlag) = 0; +}; + + +////@todo: Think about a smart way to calculate to let the compiler optimize transfer code of non-pptr classes away +//// currently we have a serializeTraits::MightContainPPtr but this should be somehow automatic! + + +/// Transfer that scans for PPtrs and optionally allows to replace them in-place. Is given a GenerateIDFunctor which maps one +/// or more existing instance IDs to new instance IDs and then crawls through an object's transfers looking for PPtr transfers. +/// Does not touch any other data. +class EXPORT_COREMODULE RemapPPtrTransfer : public TransferBase +{ +private: + + GenerateIDFunctor* m_GenerateIDFunctor; + UNITY_TEMP_VECTOR(TransferMetaFlags) m_MetaMaskStack; + TransferMetaFlags m_CachedMetaMaskStackTop; + bool m_ReadPPtrs; + +public: + + RemapPPtrTransfer (int flags, bool readPPtrs); + + void SetGenerateIDFunctor (GenerateIDFunctor* functor) { m_GenerateIDFunctor = functor; } + GenerateIDFunctor* GetGenerateIDFunctor () const { return m_GenerateIDFunctor; } + + bool IsReadingPPtr () { return m_ReadPPtrs; } + bool IsWritingPPtr () { return true; } + bool IsRemapPPtrTransfer () { return true; } + + bool DidReadLastPPtrProperty () { return true; } + + void AddMetaFlag (TransferMetaFlags flag); + + void PushMetaFlag (TransferMetaFlags flag); + void PopMetaFlag (); + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned*, const char*, TransferMetaFlags /*metaFlag*/ = kNoTransferFlags) { } + + void TransferTypelessData (unsigned, void*, TransferMetaFlags /*metaFlag*/ = kNoTransferFlags) { } + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + SInt32 GetNewInstanceIDforOldInstanceID (SInt32 oldInstanceID) { return m_GenerateIDFunctor->GenerateInstanceID (oldInstanceID, m_CachedMetaMaskStackTop);} +}; + +template<class T> +struct RemapPPtrTraits +{ + static bool IsPPtr () { return false; } + static int GetInstanceIDFromPPtr (const T& ) { AssertString ("Never"); return 0; } + static void SetInstanceIDOfPPtr (T& , SInt32) { AssertString ("Never"); } +}; + +template<class T> +struct RemapPPtrTraits<PPtr<T> > +{ + static bool IsPPtr () { return true; } + static int GetInstanceIDFromPPtr (const PPtr<T>& data) { return data.GetInstanceID (); } + static void SetInstanceIDOfPPtr (PPtr<T>& data, SInt32 instanceID) { data.SetInstanceID (instanceID); } +}; + +template<class T> +struct RemapPPtrTraits<ImmediatePtr<T> > +{ + static bool IsPPtr () { return true; } + static int GetInstanceIDFromPPtr (const ImmediatePtr<T>& data) { return data.GetInstanceID (); } + static void SetInstanceIDOfPPtr (ImmediatePtr<T>& data, SInt32 instanceID) { data.SetInstanceID (instanceID); } +}; + +template<class T> +void RemapPPtrTransfer::Transfer (T& data, const char*, TransferMetaFlags metaFlag) +{ + if (SerializeTraits<T>::MightContainPPtr ()) + { + if (metaFlag != 0) + PushMetaFlag(metaFlag); + + if (RemapPPtrTraits<T>::IsPPtr ()) + { + AssertIf (m_GenerateIDFunctor == NULL); + ANALYSIS_ASSUME (m_GenerateIDFunctor); + SInt32 oldInstanceID = RemapPPtrTraits<T>::GetInstanceIDFromPPtr (data); + SInt32 newInstanceID = GetNewInstanceIDforOldInstanceID(oldInstanceID); + + if (m_ReadPPtrs) + { + RemapPPtrTraits<T>::SetInstanceIDOfPPtr (data, newInstanceID); + } + else + { + AssertIf(oldInstanceID != newInstanceID); + } + } + else + SerializeTraits<T>::Transfer (data, *this); + + if (metaFlag != 0) + PopMetaFlag(); + } +} + +template<class T> +void RemapPPtrTransfer::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags metaFlag) +{ + Transfer(data, NULL, metaFlag); +} + + +template<class T> +void RemapPPtrTransfer::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag) +{ + if (SerializeTraits<typename T::value_type>::MightContainPPtr ()) + { + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + while (i != end) + { + Transfer (*i, "data", metaFlag); + ++i; + } + } +} + +template<class T> +void RemapPPtrTransfer::TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag) +{ + typedef typename NonConstContainerValueType<T>::value_type NonConstT; + if (SerializeTraits<NonConstT>::MightContainPPtr ()) + { + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + while (i != end) + { + NonConstT& p = (NonConstT&) (*i); + Transfer (p, "data", metaFlag); + i++; + } + } +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp b/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp new file mode 100644 index 0000000..ea4710b --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp @@ -0,0 +1,57 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" + +#if ENABLE_UNIT_TESTS + +#include "Runtime/Testing/Testing.h" +#include "Runtime/Testing/TestFixtures.h" + +SUITE (RemapPPtrTransferTests) +{ + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DoesNotTouchNonPPtrProperty) + { + UnityStr m_NonPPtrProperty = "test"; + TRANSFER (m_NonPPtrProperty); + CHECK (m_NonPPtrProperty == "test"); + } + + TEST_FIXTURE (DoesNotTouchNonPPtrPropertyTestFixture, Transfer_WithNonPPtrProperty_DoesNotChangeProperty) + { + DoRemapPPtrTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (RemapsPPtrProperty) + { + PPtr<Object> m_PPtrProperty (1234); + TRANSFER (m_PPtrProperty); + CHECK (m_PPtrProperty.GetInstanceID () == 4321); + } + + TEST_FIXTURE (RemapsPPtrPropertyTestFixture, Transfer_WithPPtrProperty_MapsToNewInstanceID) + { + AddPPtrRemap (1234, 4321); + DoRemapPPtrTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidReadLastPPtrProperty) + { + PPtr<Object> m_PPtrProperty; + TRANSFER (m_PPtrProperty); + CHECK (transfer.DidReadLastPPtrProperty ()); + } + + TEST_FIXTURE (DidReadLastPPtrPropertyTestFixture, DidReadLastPPtrProperty_WithPPtrProperty_IsTrue) + { + DoRemapPPtrTransfer (); + } + +} //TEST + +#endif // ENABLE_UNIT_TESTS + diff --git a/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp new file mode 100644 index 0000000..0fe91e8 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp @@ -0,0 +1,487 @@ +#include "UnityPrefix.h" +#include "SafeBinaryRead.h" +#include "TransferNameConversions.h" +#include "Runtime/Utilities/dynamic_bitset.h" +#include "Runtime/Utilities/algorithm_utility.h" +#include "Runtime/Utilities/InitializeAndCleanup.h" + +#define LOG_MISSING_VARIBALES 0 + +#if SUPPORT_SERIALIZED_TYPETREES + +using namespace std; + +typedef map<pair<char*, char*>, ConversionFunction*, smaller_cstring_pair> ConverterFunctions; +static ConverterFunctions* gConverterFunctions; + +namespace SafeBinaryReadManager +{ + void StaticInitialize() + { + gConverterFunctions = UNITY_NEW(ConverterFunctions,kMemSerialization); + } + void StaticDestroy() + { + UNITY_DELETE(gConverterFunctions,kMemSerialization); + } +} +static RegisterRuntimeInitializeAndCleanup s_SafeBinaryReadManagerCallbacks(SafeBinaryReadManager::StaticInitialize, SafeBinaryReadManager::StaticDestroy); + +ConversionFunction* FindConverter (const char* oldType, const char* newTypeName) +{ + pair<char*, char*> arg = make_pair(const_cast<char*> (oldType), const_cast<char*> (newTypeName)); + + ConverterFunctions::iterator found = gConverterFunctions->find (arg); + if (found == gConverterFunctions->end()) + return NULL; + + return found->second; +} + +void SafeBinaryRead::RegisterConverter (const char* oldType, const char* newType, ConversionFunction* converter) +{ + + pair<char*, char*> arg = make_pair(const_cast<char*> (oldType), const_cast<char*> (newType)); + AssertMsg (!gConverterFunctions->count (arg), "Duplicate conversion registered"); + (*gConverterFunctions)[arg] = converter; +} + +void SafeBinaryRead::CleanupConverterTable () +{ + (*gConverterFunctions).clear(); +} + +static void Walk (const TypeTree& typeTree, CachedReader& cache, SInt32* bytePosition, bool endianSwap); + +CachedReader& SafeBinaryRead::Init (const TypeTree& oldBase, int bytePosition, int byteSize, int flags) +{ + AssertIf (!m_StackInfo.empty ()); + m_OldBaseType = &oldBase; + m_BaseBytePosition = bytePosition; + AssertIf (m_BaseBytePosition < 0); + m_BaseByteSize = byteSize; + m_Flags = flags; + m_UserData = NULL; + m_DidReadLastProperty = false; + #if UNITY_EDITOR + m_TypeTreeHasChanged = false; + #endif + return m_Cache; +} + +CachedReader& SafeBinaryRead::Init (SafeBinaryRead& transfer) +{ + int newBasePosition = transfer.m_StackInfo.top ().bytePosition; + int size = transfer.m_BaseByteSize - (newBasePosition - transfer.m_BaseBytePosition); + Init (*transfer.m_StackInfo.top ().type, newBasePosition, size, transfer.m_Flags); + m_Cache.InitRead (*transfer.m_Cache.GetCacher(), transfer.m_StackInfo.top ().bytePosition, size); + m_UserData = NULL; + m_DidReadLastProperty = false; + #if UNITY_EDITOR + m_TypeTreeHasChanged = false; + #endif + + return m_Cache; +} + +SafeBinaryRead::~SafeBinaryRead () +{ + AssertIf (!m_StackInfo.empty ()); + AssertIf (!m_PositionInArray.empty ()); +} + +static void Walk (const TypeTree& typeTree, CachedReader& cache, SInt32* bytePosition, bool endianSwap) +{ + AssertIf (bytePosition == NULL); + + AssertIf((typeTree.m_ByteSize != -1 && ((typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0 || typeTree.m_Children.empty())) != (typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0)); + + if (typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0) + { + *bytePosition += typeTree.m_ByteSize; + } + else if (typeTree.m_IsArray) + { + // First child in an array is the size + // Second child is the homogenous type of the array + AssertIf (typeTree.m_Children.front ().m_Type != SerializeTraits<SInt32>::GetTypeString (NULL)); + AssertIf (typeTree.m_Children.front ().m_Name != "size"); + AssertIf (typeTree.m_Children.size () != 2); + + SInt32 arraySize, i; + cache.Read (arraySize, *bytePosition); + if (endianSwap) + SwapEndianBytes(arraySize); + + *bytePosition += sizeof (arraySize); + + const TypeTree& elementTypeTree = typeTree.m_Children.back (); + + // If the bytesize is known we can simply skip the recursive loop + if (elementTypeTree.m_ByteSize != -1 && (elementTypeTree.m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + *bytePosition += arraySize * elementTypeTree.m_ByteSize; + // Otherwise recursively Walk element typetree + else + { + for (i=0;i<arraySize;i++) + Walk (typeTree.m_Children.back (), cache, bytePosition, endianSwap); + } + } + else + { + TypeTree::TypeTreeList::const_iterator i; + for (i = typeTree.m_Children.begin (); i != typeTree.m_Children.end ();++i) + Walk (*i, cache, bytePosition, endianSwap); + } + + if (typeTree.m_MetaFlag & kAlignBytesFlag) + { + #if UNITY_EDITOR +// const TypeTree* root = &typeTree; +// while (root->m_Father != NULL) +// root = root->m_Father; +// if (root->m_Type == "MonoBehaviour") +// ErrorString("Alignment in monobehaviour???"); + #endif + *bytePosition = Align4(*bytePosition); + } +} + +// Walk through typetree and data to find the bytePosition +void SafeBinaryRead::Walk (const TypeTree& typeTree, SInt32* bytePosition) +{ + ::Walk (typeTree, m_Cache, bytePosition, ConvertEndianess()); +} + +void SafeBinaryRead::OverrideRootTypeName (const char* typeString) +{ + Assert(m_StackInfo.size() == 1); + m_StackInfo.top().currentTypeName = typeString; + #if !UNITY_RELEASE + m_StackInfo.top().currentTypeNameCheck = typeString; + #endif +} + + +int SafeBinaryRead::BeginTransfer (const char* name, const char* typeString, ConversionFunction** converter) +{ + if (converter != NULL) + *converter = NULL; + + m_DidReadLastProperty = false; + + // For the first Transfer only setup the stack to the base parameters + if (m_StackInfo.empty ()) + { + ErrorIf (name != m_OldBaseType->m_Name); + + StackedInfo newInfo; + newInfo.type = m_OldBaseType; + newInfo.bytePosition = m_BaseBytePosition; + newInfo.version = 1; + #if UNITY_EDITOR + newInfo.lookupCount = 0; + #endif + newInfo.currentTypeName = typeString; + #if !UNITY_RELEASE + newInfo.currentTypeNameCheck = typeString; + #endif + newInfo.cachedIterator = newInfo.type->begin(); + newInfo.cachedBytePosition = m_BaseBytePosition; + m_StackInfo.push(newInfo); + m_CurrentStackInfo = &m_StackInfo.top(); + + return kMatchesType; + } + + TypeTree::TypeTreeList::const_iterator c; + + StackedInfo& info = *m_CurrentStackInfo; + + const TypeTree::TypeTreeList& children = info.type->m_Children; + + // Start searching at the cached position + SInt32 newBytePosition = info.cachedBytePosition; + int count = 0; + for (c=info.cachedIterator;c != children.end ();++c) + { + if (c->m_Name == name) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + count++; + } + + #if UNITY_EDITOR + if (count > 1) + m_TypeTreeHasChanged = true; + #endif + + // Didn't find it, try again starting at the first child + if (c == children.end ()) + { + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + + // Find name conversion lookup for this type + #if !UNITY_RELEASE + DebugAssertIf(info.currentTypeName != info.currentTypeNameCheck); + #endif + const AllowNameConversion::mapped_type* nameConversion = GetAllowedNameConversions(info.currentTypeName, name); + + newBytePosition = info.bytePosition; + for (c=children.begin();c != children.end();++c) + { + if (c->m_Name == name) + break; + if (nameConversion && nameConversion->count(const_cast<char*>(c->m_Name.c_str()))) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + } + + // No child with name was found? + if (c == children.end ()) + { + #if LOG_MISSING_VARIBALES + string s ("Variable not found in old file "); + GetTypePath (m_StackInfo.top ().type, s); + s = s + " new name and type: " + name; + s = s + '(' + typeString + ")\n"; + m_OldBaseType->DebugPrint (s); + LogString (s); + #endif + + return kNotFound; + } + } + + #if UNITY_EDITOR + m_CurrentStackInfo->lookupCount++; + #endif + + info.cachedIterator = c; + info.cachedBytePosition = newBytePosition; + + /*Unoptimized version: + + // Find name in children typeTree, updating position + SInt32 newBytePosition = info.bytePosition; + + // Find name conversion lookup for this type + const AllowNameConversion::mapped_type* nameConversion = NULL; + DebugAssertIf(info.currentTypeName != info.currentTypeNameCheck); + AllowNameConversion::iterator foundNameConversion = gAllowNameConversion.find(make_pair(const_cast<char*>(info.currentTypeName), const_cast<char*>(name))); + if (foundNameConversion == gAllowNameConversion.end()) + nameConversion = &foundNameConversion->second; + + for (c=children.begin ();c != children.end ();++c) + { + if (c->m_Name == name) + break; + if (nameConversion && nameConversion->count(const_cast<char*>(c->m_Name.c_str()))) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + } + + // No child with name was found? + if (c == children.end ()) + { + #if LOG_MISSING_VARIBALES + string s ("Variable not found in old file "); + GetTypePath (m_OldType.top (), s); + s = s + " new name and type: " + name; + s = s + '(' + typeString + ")\n"; + m_OldBaseType->DebugPrint (s); + AssertStringQuiet (s); + #endif + + return kNotFound; + } + */ + + // Walk trough the already iterated array elements + if (info.type->m_IsArray && c != children.begin ()) + { + SInt32 arrayPosition = *m_CurrentPositionInArray; + + // There are no arrays in the subtree so + // we can simply use the cached bytesize + // Alignment cuts across this so use the slow path in that case + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + { + newBytePosition += c->m_ByteSize * arrayPosition; + } + // Walk through old typetree, updating position + else + { + ArrayPositionInfo& arrayInfo = m_PositionInArray.top(); + SInt32 cachedArrayPosition = 0; + if (arrayInfo.cachedArrayPosition <= arrayPosition) + { + newBytePosition = arrayInfo.cachedBytePosition; + cachedArrayPosition = arrayInfo.cachedArrayPosition; + } + + for (SInt32 i = cachedArrayPosition;i < arrayPosition;i++) + Walk (*c, &newBytePosition); + + arrayInfo.cachedArrayPosition = arrayPosition; + arrayInfo.cachedBytePosition = newBytePosition; + } + + (*m_CurrentPositionInArray)++; + } + + StackedInfo newInfo; + newInfo.type = &*c; + newInfo.bytePosition = newBytePosition; + newInfo.version = 1; + #if UNITY_EDITOR + newInfo.lookupCount = 0; + #endif + newInfo.cachedIterator = newInfo.type->begin(); + newInfo.cachedBytePosition = newBytePosition; + newInfo.currentTypeName = typeString; + #if !UNITY_RELEASE + newInfo.currentTypeNameCheck = typeString; + #endif + + m_StackInfo.push(newInfo); + m_CurrentStackInfo = &m_StackInfo.top(); + + int conversion = kNeedConversion; + // Does the type match (compare type string) + // The root type should get a transfer in any case because the type might change + // Eg. TransformComponent renamed to Transform (Typename mismatch but we still want to serialize) + if (c->m_Type == typeString || m_StackInfo.size () == 1) + { + conversion = kMatchesType; + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + conversion = kFastPathMatchesType; + } + else if (AllowTypeNameConversion (c->m_Type, typeString)) + { + conversion = kMatchesType; + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + conversion = kFastPathMatchesType; + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + } + else + { + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + } + + if (conversion == kNeedConversion && converter != NULL) + *converter = FindConverter(c->m_Type.c_str(), typeString); + + return conversion; +} + +void SafeBinaryRead::SetVersion (int version) +{ + m_CurrentStackInfo->version = version; +} + + +void SafeBinaryRead::EndTransfer () +{ + #if UNITY_EDITOR + if (m_CurrentStackInfo && m_CurrentStackInfo->lookupCount != m_CurrentStackInfo->type->m_Children.size()) + { + m_TypeTreeHasChanged = true; + } + #endif + + m_StackInfo.pop(); + if (!m_StackInfo.empty()) + { + m_CurrentStackInfo = &m_StackInfo.top(); + } + else + m_CurrentStackInfo = NULL; + + m_DidReadLastProperty = true; +} + +bool SafeBinaryRead::BeginArrayTransfer (const char* name, const char* typeString, SInt32& size) +{ + if (BeginTransfer (name, typeString, NULL) == kNotFound) + return false; + + Transfer (size, "size"); + ArrayPositionInfo info; + info.arrayPosition = 0; + info.cachedBytePosition = -1; + info.cachedArrayPosition = std::numeric_limits<SInt32>::max(); + m_PositionInArray.push (info); + m_CurrentPositionInArray = &m_PositionInArray.top().arrayPosition; + + Assert (GetActiveOldTypeTree ().m_Children.front ().m_Name == "size"); + + return true; +} + +void SafeBinaryRead::EndArrayTransfer () +{ + m_PositionInArray.pop (); + if (!m_PositionInArray.empty()) + m_CurrentPositionInArray = &m_PositionInArray.top().arrayPosition; + else + m_CurrentPositionInArray = NULL; + + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + + EndTransfer (); +} + +bool SafeBinaryRead::IsCurrentVersion () +{ + return m_CurrentStackInfo->version == m_CurrentStackInfo->type->m_Version; +} + +bool SafeBinaryRead::IsOldVersion (int version) +{ + return m_CurrentStackInfo->type->m_Version == version; +} + +bool SafeBinaryRead::IsVersionSmallerOrEqual (int version) +{ + return m_CurrentStackInfo->type->m_Version <= version; +} + +void SafeBinaryRead::TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaflag) +{ + SInt32 size; + if (!BeginArrayTransfer (name, "TypelessData", size)) + { + *byteSize = 0; + return; + } + // We can only transfer the array if the size was transferred as well + AssertIf (GetActiveOldTypeTree ().m_Children.front ().m_Name != "size"); + + *byteSize = size; + + EndArrayTransfer (); +} + +void SafeBinaryRead::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + if (copyData == NULL || byteSize == 0) return; + + m_Cache.Read (copyData, byteSize); +} + +#endif // SUPPORT_SERIALIZED_TYPETREES diff --git a/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h new file mode 100644 index 0000000..e6cd5a3 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h @@ -0,0 +1,290 @@ +#ifndef SAFEBINARYREAD_H +#define SAFEBINARYREAD_H + +#include "Configuration/UnityConfigure.h" + +#if SUPPORT_SERIALIZED_TYPETREES + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + +#include <stack> +class dynamic_bitset; +class SafeBinaryRead; + +#define LOG_CONVERTING_VARIBALES 0 +#define LOG_MISSING_VARIBALES 0 + +typedef bool ConversionFunction (void* inData, SafeBinaryRead& transfer); + +class EXPORT_COREMODULE SafeBinaryRead : public TransferBase +{ + CachedReader m_Cache; + SInt32 m_BaseBytePosition; + SInt32 m_BaseByteSize; + + const TypeTree* m_OldBaseType; + #if UNITY_EDITOR + bool m_TypeTreeHasChanged; + #endif + + enum { kNotFound = 0, kMatchesType = 1, kFastPathMatchesType = 2, kNeedConversion = -1 }; + + struct StackedInfo + { + const TypeTree* type; /// The type tree of the old type we are reading data from + const char* currentTypeName; /// The name of the type we are currently reading (This is the new type name and not from the stored data) + int bytePosition;/// byte position of that element + int version; /// current version (This is the new version and not from the stored data) + + int cachedBytePosition; /// The cached byte position of the last visited child + TypeTree::const_iterator cachedIterator; /// The cached iterator of the last visited child + #if UNITY_EDITOR + int lookupCount; // counts number of looks, used to determine if the typetree matches + #endif + + #if !UNITY_RELEASE + std::string currentTypeNameCheck;/// For debugging purposes in case someone changes the typename string while still reading! + #endif + + }; + + StackedInfo* m_CurrentStackInfo; + SInt32* m_CurrentPositionInArray; + std::stack<StackedInfo> m_StackInfo; + + struct ArrayPositionInfo + { + SInt32 arrayPosition; + SInt32 cachedBytePosition; + SInt32 cachedArrayPosition; + }; + + std::stack<ArrayPositionInfo> m_PositionInArray;// position in an array + + bool m_DidReadLastProperty; + + friend class MonoBehaviour; + +public: + + CachedReader& Init (const TypeTree& oldBase, int bytePosition, int byteSize, int flags); + CachedReader& Init (SafeBinaryRead& transfer); + ~SafeBinaryRead (); + + void SetVersion (int version); + bool IsCurrentVersion (); + bool IsOldVersion (int version); + bool IsVersionSmallerOrEqual (int version); + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool IsReadingBackwardsCompatible () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return m_Flags & kSwapEndianess; } + + bool DidReadLastProperty () { return m_DidReadLastProperty; } + bool DidReadLastPPtrProperty () { return m_DidReadLastProperty; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + /// In order to transfer typeless data (Read: transfer data real fast) + /// Call TransferTypeless. You have to always do this. Even for a proxytransfer. Then when you want to access the datablock. + /// Call TransferTypelessData + /// On return: + /// When reading bytesize will contain the size of the data block that should be read, + /// when writing bytesize has to contain the size of the datablock. + /// MarkerID will contain an marker which you have to give TransferTypelessData when you want to start the actual transfer. + /// optional: A serializedFile will be seperated into two chunks. One is the normal object data. (It is assumed that they are all relatively small) + /// So caching them makes a lot of sense. Big datachunks will be stored in another part of the file. + /// They will not be cached but usually read directly into the allocated memory, probably reading them asynchronously + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + // markerID is the id that was given by TransferTypeless. + // byteStart is the bytestart relative to the beginning of the typeless data + // copyData is a pointer to where the data will be written or read from + /// optional: if metaFlag is kTransferBigData the data will be optimized into seperate blocks, + void TransferTypelessData (unsigned byteSize, void* copyData, int metaData = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + bool GetTransferFileInfo(unsigned* position, const char** filePath) const; + + const TypeTree& GetActiveOldTypeTree () { return *m_CurrentStackInfo->type; } + + static void RegisterConverter (const char* oldType, const char* newType, ConversionFunction* converter); + static void CleanupConverterTable (); + + #if UNITY_EDITOR + /// Returns if the typetree is different from what was loaded in. + /// Currently this is incomplete. Arrays will always return true. + bool HasDifferentTypeTree () + { + return m_TypeTreeHasChanged; + } + #endif + +private: + + // BeginTransfer / EndTransfer + int BeginTransfer (const char* name, const char* typeString, ConversionFunction** converter); + bool BeginArrayTransfer (const char* name, const char* typeString, SInt32& size); + + // Override the root type name, this is used by scripts that can only determine the class type name after the mono class has actually been loaded + void OverrideRootTypeName (const char* typeString); + + void EndTransfer (); + void EndArrayTransfer (); + + void Walk (const TypeTree& typeTree, SInt32* bytePosition); +}; + +template<class T>inline +void SafeBinaryRead::TransferBasicData (T& data) +{ + m_Cache.Read (data, m_CurrentStackInfo->bytePosition); + if (ConvertEndianess()) + { + SwapEndianBytes (data); + } +} + +template<class T> inline +void SafeBinaryRead::TransferSTLStyleArray (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + if (!BeginArrayTransfer ("Array", "Array", size)) + return; + + SerializeTraits<T>::ResizeSTLStyleArray (data, size); + + typename T::iterator i; + typename T::iterator end = data.end (); + if (size != 0) + { + int conversion = BeginTransfer ("data", SerializeTraits<typename T::value_type>::GetTypeString(&*data.begin()), NULL); + int elementSize = m_CurrentStackInfo->type->m_ByteSize; + *m_CurrentPositionInArray = 0; + // If the data types are matching and element size can be determined + // then we fast path the whole thing and skip all the duplicate stack walking + if (conversion == kFastPathMatchesType) + { + int basePosition = m_CurrentStackInfo->bytePosition; + + for (i = data.begin ();i != end;++i) + { + int currentBytePosition = basePosition + (*m_CurrentPositionInArray) * elementSize; + m_CurrentStackInfo->cachedBytePosition = currentBytePosition; + m_CurrentStackInfo->bytePosition = currentBytePosition; + m_CurrentStackInfo->cachedIterator = m_CurrentStackInfo->type->begin(); + (*m_CurrentPositionInArray)++; + SerializeTraits<typename T::value_type>::Transfer (*i, *this); + } + EndTransfer(); + } + // Fall back to converting variables + else + { + EndTransfer(); + for (i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } + } + + EndArrayTransfer (); +} + +template<class T> inline +void SafeBinaryRead::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + if (!BeginArrayTransfer ("Array", "Array", size)) + return; + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typename NonConstContainerValueType<T>::value_type p; + + data.clear (); + for (int i=0;i<size;i++) + { + Transfer (p, "data"); + data.insert (p); + } + EndArrayTransfer (); +} + +template<class T> inline +void SafeBinaryRead::TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags) +{ + ConversionFunction* converter; + int conversion = BeginTransfer (name, typeName, &converter); + if (conversion == kNotFound) + return; + + if (conversion >= kMatchesType) + SerializeTraits<T>::Transfer (data, *this); + // Try conversion + else + { + bool success = false; + if (converter != NULL) + success = converter (&data, *this); + + #if LOG_CONVERTING_VARIBALES + { + string s ("Converting variable "); + if (success) + s += " succeeded "; + else + s += " failed "; + + GetTypePath (m_OldType.top (), s); + s = s + " new type: "; + s = s + " new type: (" + SerializeTraits<T>::GetTypeString () + ")\n"; + m_OldBaseType->DebugPrint (s); + AssertStringQuiet (s); + } + #endif + } + EndTransfer (); +} + +template<class T> inline +void SafeBinaryRead::Transfer (T& data, const char* name, TransferMetaFlags) +{ + TransferWithTypeString(data, name, SerializeTraits<T>::GetTypeString(&data), kNoTransferFlags); +} + +#else + +namespace SafeBinaryReadManager +{ + inline void StaticInitialize(){}; + inline void StaticDestroy(){}; +} + +class SafeBinaryRead +{ +public: + static void RegisterAllowTypeNameConversion (const char* oldTypeName, const char* newTypeName) { } + static void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName) { } + static void CleanupConverterTable() { } +}; + +#endif +#endif diff --git a/Runtime/Serialize/TransferFunctions/SerializeTransfer.h b/Runtime/Serialize/TransferFunctions/SerializeTransfer.h new file mode 100644 index 0000000..f96d7d5 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SerializeTransfer.h @@ -0,0 +1,14 @@ +#ifndef SERIALIZETRANSFER_H +#define SERIALIZETRANSFER_H + +#if SUPPORT_TEXT_SERIALIZATION +#include "YAMLRead.h" +#include "YAMLWrite.h" +#endif +#include "RemapPPtrTransfer.h" +#include "StreamedBinaryRead.h" +#include "ProxyTransfer.h" +#include "SafeBinaryRead.h" +#include "StreamedBinaryWrite.h" + +#endif diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp new file mode 100644 index 0000000..4daa8fb --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp @@ -0,0 +1,54 @@ +#include "UnityPrefix.h" +#include "StreamedBinaryRead.h" + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::Align () +{ + m_Cache.Align4Read(); +} + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/) +{ + SInt32 size; + Transfer (size, "size"); + *byteSize = size; +} + +// markerID is the id that was given by TransferTypeless. +// optional copyData: is a pointer to where the data will be written or read from +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + if (byteSize == 0) + return; + + if (copyData == NULL) + { + // seek byte + m_Cache.Skip(byteSize); + } + else + m_Cache.Read (copyData, byteSize); + Align(); +} + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::ReadDirect (void* data, int byteSize) +{ + AssertIf (kSwapEndianess); + m_Cache.Read (data, byteSize); +} + + +template void StreamedBinaryRead<true>::ReadDirect (void* data, int byteSize); +template void StreamedBinaryRead<false>::ReadDirect (void* data, int byteSize); + +template void StreamedBinaryRead<true>::Align(); +template void StreamedBinaryRead<false>::Align(); + +template void StreamedBinaryRead<true>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); +template void StreamedBinaryRead<false>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); + +template void StreamedBinaryRead<true>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); +template void StreamedBinaryRead<false>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h new file mode 100644 index 0000000..81a015c --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h @@ -0,0 +1,174 @@ +#ifndef STREAMEDBINARYREAD_H +#define STREAMEDBINARYREAD_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + + +template<bool kSwapEndianess> +class StreamedBinaryRead : public TransferBase +{ + CachedReader m_Cache; + + friend class MonoBehaviour; + +public: + + CachedReader& Init (int flags) { m_UserData = NULL; m_Flags = flags; return m_Cache; } + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return kSwapEndianess; } + + bool DidReadLastProperty () { return true; } + bool DidReadLastPPtrProperty () { return true; } + + void EnableResourceImage (ActiveResourceImage targetResourceImage) { m_Cache.BeginResourceImage(targetResourceImage); } + bool ReadStreamingInfo(StreamingInfo* streamingInfo); + + bool ShouldChannelOverride (); + CachedReader& GetCachedReader () { return m_Cache; } + + const char* GetSerializedFilePathName() { return m_Cache.GetSerializedFilePathName(); } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + // markerID is the id that was given by TransferTypeless. + // optional copyData: is a pointer to where the data will be written or read from + void TransferTypelessData (unsigned byteSize, void* copyData, int metaData = 0); + + /// Reads byteSize bytes into data. This may onle be used if UseOptimizedReading returns true. + void EXPORT_COREMODULE ReadDirect (void* data, int byteSize); + + void EXPORT_COREMODULE Align (); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); +}; + +template <bool kSwapEndianess> +bool StreamedBinaryRead<kSwapEndianess>::ReadStreamingInfo(StreamingInfo* streamingInfo) +{ + Assert(streamingInfo != NULL); + + if (!m_Cache.IsReadingResourceImage()) + return false; + + // Read the size & offset values from the serialized file + // The size & offset describes where the data is in the streamed file + UInt32 offset, size; + Transfer (size, "ri_size"); + Transfer (offset, "ri_offset"); + + m_Cache.GetStreamingInfo (offset, size, streamingInfo); + return true; +} + + + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferSTLStyleArray (T& data, TransferMetaFlags /*metaFlags*/) +{ + if (m_Cache.IsReadingResourceImage()) + { + // Read the size & offset from the serialized file + UInt32 offset, size; + Transfer (size, "ri_size"); + Transfer (offset, "ri_offset"); + + // Fetch the pointer from the pre-loaded resource image. + unsigned bufferSize = sizeof (typename T::value_type) * size; + UInt8* buffer = m_Cache.FetchResourceImageData (offset, bufferSize); + SerializeTraits<T>::resource_image_assign_external (data, buffer, buffer + bufferSize); + + m_Cache.EndResourceImage(); + } + else + { + SInt32 size; + Transfer (size, "size"); + + SerializeTraits<T>::ResizeSTLStyleArray (data, size); + + if (!kSwapEndianess && SerializeTraits<typename T::value_type>::AllowTransferOptimization () && SerializeTraits<T>::IsContinousMemoryArray ()) + { + //AssertIf (size != distance (data.begin (), data.end ())); + if( size != 0 ) + ReadDirect (&*data.begin (), size * sizeof (typename T::value_type)); + } + else + { + typename T::iterator i; + typename T::iterator end = data.end (); + //AssertIf (size != distance (data.begin (), end)); + for (i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } + } +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size; + Transfer (size, "size"); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typename NonConstContainerValueType<T>::value_type p; + + data.clear (); + for (int i=0;i<size;i++) + { + Transfer (p, "data"); + data.insert (p); + } +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::Transfer (T& data, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryRead<kSwapEndianess>::TransferBasicData (T& data) +{ + AssertIf (sizeof (T) > 8); + m_Cache.Read (data); + if (kSwapEndianess) + { + SwapEndianBytes (data); + } +} +#endif + + + diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp new file mode 100644 index 0000000..a7699e9 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp @@ -0,0 +1,76 @@ +#include "UnityPrefix.h" +#include "StreamedBinaryWrite.h" +#include "Configuration/UnityConfigure.h" + +template <bool kSwapEndianess> +CachedWriter& StreamedBinaryWrite<kSwapEndianess>::Init (int flags, BuildTargetSelection target) +{ + m_Flags = flags; + m_UserData = NULL; + m_Target = target; + +#if UNITY_EDITOR && CHECK_SERIALIZE_ALIGNMENT + m_Cache.SetCheckSerializeAlignment(true); + #endif + return m_Cache; +} + +template <bool kSwapEndianess> +CachedWriter& StreamedBinaryWrite<kSwapEndianess>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag) +{ + m_Flags = flags; + m_Target = target; + m_Cache = cachedWriter; + m_UserData = NULL; + + #if UNITY_EDITOR + m_BuildUsageTag = buildUsageTag; + #endif + +#if UNITY_EDITOR && CHECK_SERIALIZE_ALIGNMENT + m_Cache.SetCheckSerializeAlignment(true); +#endif + return m_Cache; +} + +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::Align () +{ + m_Cache.Align4Write(); +} + + +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::TransferTypeless (unsigned* byteSize, const char* /* name*/, TransferMetaFlags/* metaFlag*/) +{ + SInt32 size = *byteSize; + Transfer (size, "size"); +} + +// markerID is the id that was given by TransferTypeless. +// byteStart is +// optional temporaryDataHandle: temporaryDataHandle is a handle to the data +// optional copyData: is a pointer to where the data will be written or read from +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::TransferTypelessData (unsigned byteSize, void* copyData, int/* metaData*/) +{ + AssertIf(copyData == NULL && byteSize != 0); + m_Cache.Write (copyData, byteSize); + Align(); +} + + +template CachedWriter& StreamedBinaryWrite<false>::Init (int flags, BuildTargetSelection target); +template CachedWriter& StreamedBinaryWrite<true>::Init (int flags, BuildTargetSelection target); + +template CachedWriter& StreamedBinaryWrite<false>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); +template CachedWriter& StreamedBinaryWrite<true>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); + +template void StreamedBinaryWrite<false>::Align (); +template void StreamedBinaryWrite<true>::Align (); + +template void StreamedBinaryWrite<false>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); +template void StreamedBinaryWrite<true>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); + +template void StreamedBinaryWrite<false>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); +template void StreamedBinaryWrite<true>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h new file mode 100644 index 0000000..b5d9074 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h @@ -0,0 +1,187 @@ +#ifndef STREAMEDBINARYWRITE_H +#define STREAMEDBINARYWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + +template<bool kSwapEndianess> +class EXPORT_COREMODULE StreamedBinaryWrite : public TransferBase +{ + CachedWriter m_Cache; + BuildTargetSelection m_Target; + + #if UNITY_EDITOR + BuildUsageTag m_BuildUsageTag; + #endif + friend class MonoBehaviour; + +public: + + CachedWriter& Init (int flags, BuildTargetSelection target); + CachedWriter& Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); + + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return kSwapEndianess; } + bool IsWritingGameReleaseData () + { + return IsSerializingForGameRelease (); + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + #if UNITY_EDITOR + if (platform == kBuildAnyPlayerData) + return m_Target.platform >= kBuildValidPlayer; + else + return m_Target.platform == platform; + #else + return false; + #endif + } + + #if UNITY_EDITOR + BuildUsageTag GetBuildUsage () + { + return m_BuildUsageTag; + } + #endif + + BuildTargetSelection GetBuildingTarget () { return m_Target; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void EnableResourceImage (ActiveResourceImage targetResourceImage) + { + #if UNITY_EDITOR + m_Cache.BeginResourceImage (targetResourceImage); + #endif + } + + /// In order to transfer typeless data (Read: transfer data real fast) + /// Call TransferTypeless. You have to always do this. Even for a proxytransfer. Then when you want to access the datablock. + /// Call TransferTypelessData + /// On return: + /// When reading bytesize will contain the size of the data block that should be read, + /// when writing bytesize has to contain the size of the datablock. + /// MarkerID will contain an marker which you have to give TransferTypelessData when you want to start the actual transfer. + /// optional: A serializedFile will be seperated into two chunks. One is the normal object data. (It is assumed that they are all relatively small) + /// So caching them makes a lot of sense. Big datachunks will be stored in another part of the file. + /// They will not be cached but usually read directly into the allocated memory, probably reading them asynchronously + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + // markerID is the id that was given by TransferTypeless. + // copyData is a pointer to where the data will be written or read from + void TransferTypelessData (unsigned byteSize, void* copyData, int metaFlag = 0); + + bool GetTransferFileInfo(unsigned* position, const char** filePath) const; + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + void Align (); + + CachedWriter& GetCachedWriter() { return m_Cache; } +}; + + + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferSTLStyleArray (T& data, TransferMetaFlags /*metaFlags*/) +{ + #if UNITY_EDITOR + if (m_Cache.IsWritingResourceImage()) + { + // Grab the offset where the resourceImage is currently at + UInt32 offsetInResourceImage = m_Cache.GetPosition(); + + // Write the actual data to the resource image + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + Transfer (*i, "data"); + + Assert (m_Cache.IsWritingResourceImage()); + m_Cache.EndResourceImage (); + Assert (!m_Cache.IsWritingResourceImage()); + + UInt32 size = data.size (); + + // Writ ethe size & offset to the serialized file + Transfer (size, "ri_size"); + Transfer (offsetInResourceImage, "ri_offset"); + } + else + #endif + { + SInt32 size = data.size (); + Transfer (size, "size"); + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } +} + + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + Transfer (size, "size"); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + { + non_const_value_type& p = (non_const_value_type&)(*i); + Transfer (p, "data"); + } +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::Transfer (T& data, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferBasicData (T& data) +{ + if (kSwapEndianess) + { + T temp = data; + SwapEndianBytes (temp); + m_Cache.Write (temp); + } + else + m_Cache.Write (data); +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/TransferBase.h b/Runtime/Serialize/TransferFunctions/TransferBase.h new file mode 100644 index 0000000..aae527a --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferBase.h @@ -0,0 +1,177 @@ +#ifndef TRANSFERBASE_H +#define TRANSFERBASE_H + +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + + +struct ReduceCopyData; +struct StreamingInfo; + + +class EXPORT_COREMODULE TransferBase +{ +public: + + TransferBase () + : m_Flags (0) + , m_UserData (NULL) {} + + /// @name Predicates + /// @{ + + /// Get the TransferInstructionFlags. + /// Commonly used to special case transfer functions for specific operations. + int GetFlags () { return m_Flags; } + + /// If true, the transfer is reading data from a source. (Could be fread from a file or reading from a block of memory) + /// @note There are transfers for which neither IsReading() nor IsWriting() is true (for example when generating a typetree). + /// IsReading is NOT the inverse of IsWriting. + bool IsReading () { return false; } + + /// If true, the transfer reads PPtrs (Object references) + /// This is true when reading from a memory stream or file but also when using RemapPPtrTransfer (A generic way of iterating all object references) + bool IsReadingPPtr () { return false; } + + /// Whether the last Transfer() resulted in a value store, i.e. had actual data + /// transfered from the stream. + /// It is important to use this function instead of IsReading when + /// When reading from a stream that does not define all the data, the desired behaviour is that default values from constructor are fully preserved. + /// All transfer functions do this internally (transferred properties are left untouched when the data does not exist for example in a Yaml file) + /// But when the serialized property needs to be manually converted in the Transfer function, then it is important to check if the value was actually read. + /// CODE EXAMPLE: + /// bool enabled; + /// if (transfer.IsWriting ()) + /// enabled = m_Flags == 1; + /// TRANSFER (enabled); + /// if (transfer.DidReadLastProperty ()) + /// m_Flags = enabled ? 1 : 0; + bool DidReadLastProperty () const { return false; } + + /// Same as DidReadLastProperty, but only returns true when reading PPtr properties. + /// A compile time optimization necessary for removing generated code by RemapPPtrTransfer. + bool DidReadLastPPtrProperty () const { return false; } + + + /// If true, the transfer is writing out data. + bool IsWriting () { return false; } + /// If true, the transfer is writing out PPtr data. This is true when writing to a memory stream or file, but also when using RemapPPtrTransfer. + bool IsWritingPPtr () { return false; } + + /// Are we reading data from a data source that is not guaranteed to have the same data layout as the Transfer function. + /// eg. StreamedBinaryRead always returns false. YamlRead & SafeBinaryRead return true. + bool IsReadingBackwardsCompatible () { return false; } + + /// When writing or reading from disk we need to translate instanceID + /// to LocalIdentifierInFile & LocalSerializedFileIndex or in the case of Yaml files, guids + LocalIdentifierInFile. + /// This returns true when remapping of the instanceID should be performed. + bool NeedsInstanceIDRemapping () { return false; } + + /// Are we transferring data with endianess swapping. (We might neither endianess swap on write or read based on IsReading / IsWriting) + /// The endianess conversion is done by the TransferBackend, but there are some special cases where you might want to handle it yourself. + /// (For example a texture data is transferred a single UInt8* array, so all endianess swapping is the responsibiltiy of the texture transfer function.) + bool ConvertEndianess () { return false; } + + /// Are we reading/writing a .meta file (Asset importers use it to differentiate reading/writing of a Library/metadata cached file. ) + /// @TODO: We should rename Library/metadata to Library/cachedata and cleanup the usage of metadata vs .meta file. It is confusing. + bool AssetMetaDataOnly () { return false; } + + /// Is this a RemapPPtrTransfer backend. Commonly used to do very specialized code when generating dependencies using RemapPPtrTransfer. + bool IsRemapPPtrTransfer () { return false; } + + /// Return true if we are writing the data for a player. + bool IsWritingGameReleaseData () { return false; } + + /// Are we serializing data for use by the player. + /// This includes reading/writing/generating typetrees. And can be when creating data from the editor for player or when simply reading/writing data in the player. + /// Commonly used to not serialize data that does not exist in the player. + bool IsSerializingForGameRelease () + { + #if UNITY_EDITOR + return m_Flags & kSerializeGameRelease; + #else + return true; + #endif + } + + /// @} + + /// @name Build Targets + /// @{ + + /// Returns true in the editor when writing the data for a player of the specified target platform. + bool IsBuildingTargetPlatform (BuildTargetPlatform) { return false; } + /// Returns the target platform we are building for. Only returns the target platform when writing data. + BuildTargetSelection GetBuildingTarget () { return BuildTargetSelection::NoTarget (); } + + #if UNITY_EDITOR + /// BuildUsageTag carries information generated by the build process about the object being serialized. + /// For example the buildpipeline might instruct the transfer system to strip normals and tangents from a Mesh, + /// because it knows that no renderers & materials in all scenes actually use them. + BuildUsageTag GetBuildUsage () { return BuildUsageTag (); } + #endif + + /// @} + + /// @name Versioning + /// @{ + + /// Sets the "version of the class currently transferred" + void SetVersion (int) {} + + /// Returns if the transferred data's version is the version used by the source code + bool IsVersionSmallerOrEqual (int /*version*/) { return false; } + + /// Deprecated: use IsVersionSmallerOrEqual instead. + bool IsOldVersion (int /*version*/) { return false; } + bool IsCurrentVersion () { return true; } + + /// @} + + /// @name Transfers + /// @{ + + /// Alignment in the serialization system is done manually. + /// The serialization system only ever cares about 4 byte alignment. + /// When writing data that has an alignment of less than 4 bytes, followed by data that has 4 byte alignment, + /// then Align must be called before the 4 byte aligned data. + /// TRANSFER (1byte); + /// TRANSFER (1byte); + /// transfer.Align (); + /// TRANSFER (4byte); + void Align () {} + + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void TransferBasicData (T&) { } + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void TransferPtr (bool, ReduceCopyData*) {} + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void ReduceCopy (const ReduceCopyData&){} + /// @} + + /// user data. + void* GetUserData () { return m_UserData; } + void SetUserData (void* userData) { m_UserData = userData; } + + void AddMetaFlag(int /*mask*/) {} + + /// Deprecated + void BeginMetaGroup (std::string /*name*/) {} + void EndMetaGroup () {} + void EnableResourceImage (ActiveResourceImage /*targetResourceImage*/) {} + bool ReadStreamingInfo (StreamingInfo* /*streamingInfo*/) { return false; } + bool NeedNonCriticalMetaFlags () { return false; } + +protected: + + int m_Flags; + void* m_UserData; +}; + +#endif // !TRANSFER_BASE diff --git a/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp b/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp new file mode 100644 index 0000000..35ae03f --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp @@ -0,0 +1,76 @@ +#include "UnityPrefix.h" +#include "TransferNameConversions.h" +#include "Runtime/Utilities/InitializeAndCleanup.h" + + +TranferNameConversionsManager::TranferNameConversionsManager() +{ + m_AllowTypeNameConversions = UNITY_NEW(AllowTypeNameConversions,kMemSerialization); + m_AllowNameConversion = UNITY_NEW(AllowNameConversion,kMemSerialization); +} + +TranferNameConversionsManager::~TranferNameConversionsManager() +{ + UNITY_DELETE(m_AllowTypeNameConversions,kMemSerialization); + UNITY_DELETE(m_AllowNameConversion,kMemSerialization); +} + +TranferNameConversionsManager* TranferNameConversionsManager::s_Instance = NULL; +void TranferNameConversionsManager::StaticInitialize() +{ + s_Instance = UNITY_NEW_AS_ROOT(TranferNameConversionsManager, kMemManager, "SerializationBackwardsCompatibility", ""); +} +void TranferNameConversionsManager::StaticDestroy() +{ + UNITY_DELETE(s_Instance, kMemManager); +} +static RegisterRuntimeInitializeAndCleanup s_TranferNameConversionsManagerCallbacks(TranferNameConversionsManager::StaticInitialize, TranferNameConversionsManager::StaticDestroy); + +bool AllowTypeNameConversion (const UnityStr& oldType, const char* newTypeName) +{ + pair<AllowTypeNameConversions::iterator, AllowTypeNameConversions::iterator> range; + range = GetTranferNameConversionsManager().m_AllowTypeNameConversions->equal_range (const_cast<char*>(oldType.c_str())); + for (;range.first != range.second;range.first++) + { + if (strcmp(range.first->second, newTypeName) == 0) + return true; + } + + // Special support for Mono PPtr's + // With Unity 1.6 MonoBehaviour pointers have a special prefix and keep the class name in the PPtr. [ PPtr<$MyClass> ] + // With Unity 1.5.1 it was simply PPtr<MonoBehaviour>. This made correct typechecking unneccessarily hard. + // Here we provide backwards compatibility with the old method. + if (strncmp("PPtr<$", newTypeName, 6) == 0) + { + if (oldType.find("PPtr<") == 0) + return true; + } + + return false; +} + +const AllowNameConversion::mapped_type* GetAllowedNameConversions (const char* type, const char* name) +{ + const AllowNameConversion::mapped_type* nameConversion = NULL; + AllowNameConversion::iterator foundNameConversion = GetTranferNameConversionsManager().m_AllowNameConversion->find(make_pair(const_cast<char*>(type), const_cast<char*>(name))); + if (foundNameConversion != GetTranferNameConversionsManager().m_AllowNameConversion->end()) + nameConversion = &foundNameConversion->second; + return nameConversion; +} + +void RegisterAllowTypeNameConversion (const char* from, const char* to) +{ + GetTranferNameConversionsManager().m_AllowTypeNameConversions->insert(make_pair(const_cast<char*>(from), const_cast<char*>(to))); +} + +void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName) +{ + AllowNameConversion::mapped_type& allowed = (*GetTranferNameConversionsManager().m_AllowNameConversion)[make_pair(const_cast<char*>(type), const_cast<char*>(newName))]; + allowed.insert (const_cast<char*>(oldName)); +} + +void ClearTypeNameConversion() +{ + GetTranferNameConversionsManager().m_AllowTypeNameConversions->clear(); + GetTranferNameConversionsManager().m_AllowNameConversion->clear(); +} diff --git a/Runtime/Serialize/TransferFunctions/TransferNameConversions.h b/Runtime/Serialize/TransferFunctions/TransferNameConversions.h new file mode 100644 index 0000000..3082f8f --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferNameConversions.h @@ -0,0 +1,56 @@ +#ifndef TRANSFERNAMECONVERSIONS_H +#define TRANSFERNAMECONVERSIONS_H + +#include "Runtime/Utilities/CStringHash.h" +#include "Runtime/Modules/ExportModules.h" + +using namespace std; + +struct smaller_cstring_pair : std::binary_function<std::pair<char*, char*>, std::pair<char*, char*>, std::size_t> +{ + bool operator () (pair<char*, char*> lhs, pair<char*, char*> rhs) const + { + int first = strcmp (lhs.first, rhs.first); + if (first != 0) + return first < 0; + else + return strcmp (lhs.second, rhs.second) < 0; + } +}; + +typedef std::multimap<char*, char*, smaller_cstring> AllowTypeNameConversions; +typedef std::map<std::pair<char*, char*>, set<char*, smaller_cstring>, smaller_cstring_pair> AllowNameConversion; + +class TranferNameConversionsManager +{ +public: + AllowTypeNameConversions* m_AllowTypeNameConversions; + AllowNameConversion* m_AllowNameConversion; + + TranferNameConversionsManager(); + ~TranferNameConversionsManager(); + + static TranferNameConversionsManager* s_Instance; + static void StaticInitialize(); + static void StaticDestroy(); +}; +inline TranferNameConversionsManager& GetTranferNameConversionsManager() { return *TranferNameConversionsManager::s_Instance; } + + +/// Allows type name conversion from oldTypeName to newTypeName(The passed strings will not be copied so you can only pass in constant strings) +/// (Useful for depracating types -> RegisterAllowTypeNameConversion ("UniqueIdentifier", "GUID");) +/// "UniqueIdentifier" can now be renamed to "GUID" and serialization will just work! +void RegisterAllowTypeNameConversion (const char* oldTypeName, const char* newTypeName); + +/// Allows name conversion from oldName to newName? (The passed strings will not be copied so you can only pass in constant strings) +/// (Useful for deprecating names -> m_NewPosition will now load from m_DeprecatedPosition in an old serialized file +/// RegisterAllowNameConversion (MyClass::GetClassStringStatic(), "m_DeprecatedPosition", "m_NewPosition"); +EXPORT_COREMODULE void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName); + +const AllowNameConversion::mapped_type* GetAllowedNameConversions (const char* type, const char* name); + +bool AllowTypeNameConversion (const UnityStr& oldType, const char* newTypeName); + +void ClearTypeNameConversion (); + +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLRead.cpp b/Runtime/Serialize/TransferFunctions/YAMLRead.cpp new file mode 100644 index 0000000..4a9ca42 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLRead.cpp @@ -0,0 +1,238 @@ +#include "UnityPrefix.h" +#include "YAMLRead.h" +#include "../FileCache.h" + +int YAMLRead::GetDataVersion () +{ + if (m_Versions.back() == -1) + { + yaml_node_t *node = m_CurrentNode; + int i = m_MetaParents.size(); + do + { + yaml_node_t *versionNode = GetValueForKey(node, "serializedVersion"); + if (versionNode) + { + Assert (versionNode->type == YAML_SCALAR_NODE); + sscanf ((char*)versionNode->data.scalar.value, "%d", &m_Versions.back()); + return m_Versions.back(); + } + // If "serializedVersion" is not found, look for "importerVersion" for backwards compatibility. + versionNode = GetValueForKey(node, "importerVersion"); + if (versionNode) + { + Assert (versionNode->type == YAML_SCALAR_NODE); + sscanf ((char*)versionNode->data.scalar.value, "%d", &m_Versions.back()); + return m_Versions.back(); + } + if (i>0) + node = m_MetaParents[--i]; + else + node = NULL; + } + while (node != NULL); + m_Versions.back() = 1; + } + return m_Versions.back(); +} + +yaml_node_t *YAMLRead::GetValueForKey (yaml_node_t* parentNode, const char* keystr) +{ + if (parentNode && parentNode->type == YAML_MAPPING_NODE) + { + // The code below does not handle empty yaml arrays. + if (parentNode->data.mapping.pairs.top == parentNode->data.mapping.pairs.start) + return NULL; + + yaml_node_pair_t* start; + if (m_CachedIndex < parentNode->data.mapping.pairs.top + && m_CachedIndex >= parentNode->data.mapping.pairs.start) + start = m_CachedIndex; + else + start = parentNode->data.mapping.pairs.start; + + yaml_node_pair_t* top = parentNode->data.mapping.pairs.top; + yaml_node_pair_t* i = start; + + do + { + yaml_node_pair_t* next = i+1; + if (next == top) + next = parentNode->data.mapping.pairs.start; + + yaml_node_t* key = yaml_document_get_node(m_ActiveDocument, i->key); + if (key == NULL) + { + // I've seen a crash bug report with no repro, indicating that this is happening. + // If you ever get this error and can repro it, let me know! jonas. + ErrorString ("YAML Node is NULL!\n"); + } + else + { + Assert (key->type == YAML_SCALAR_NODE); + + if (strcmp((char*)key->data.scalar.value, keystr) == 0) + { + m_CachedIndex = next; + return yaml_document_get_node(m_ActiveDocument, i->value); + } + } + i = next; + } + while (i != start); + } + return NULL; +} + + +void YAMLRead::Init(int flags, yaml_read_handler_t *handler, std::string *debugFileName, int debugLineCount) +{ + m_UserData = NULL; + m_CurrentVersion = 0; + m_Flags = flags; + m_CachedIndex = NULL; + m_ReadHandler = handler; + + yaml_parser_t parser; + + memset(&parser, 0, sizeof(parser)); + memset(&m_Document, 0, sizeof(m_Document)); + + if (!yaml_parser_initialize(&parser)) + { + ErrorString("Could not initialize yaml parser\n"); + return; + } + + yaml_parser_set_input(&parser, handler, this ); + yaml_parser_load(&parser, &m_Document); + + if (parser.error != YAML_NO_ERROR) + { + if (debugFileName != NULL) + { + ErrorStringMsg("Unable to parse file %s: [%s] at line %d\n", debugFileName->c_str(), parser.problem, debugLineCount + (int)parser.problem_mark.line); + } + else + { + ErrorStringMsg("Unable to parse YAML file: [%s] at line %d\n", parser.problem, debugLineCount + (int)parser.problem_mark.line); + } + } + + yaml_parser_delete(&parser); + + m_Versions.push_back(-1); + m_CurrentNode = yaml_document_get_root_node(&m_Document); + m_ActiveDocument = &m_Document; + m_DidReadLastProperty = false; +} + +YAMLRead::YAMLRead (yaml_document_t* yamlDocument, int flags) +: m_ReadHandler (NULL) +, m_ActiveDocument (yamlDocument) +, m_CurrentVersion (0) +, m_CachedIndex (0) +, m_DidReadLastProperty (false) +{ + m_Flags = flags; + memset(&m_Document, 0, sizeof(m_Document)); + m_Versions.push_back(-1); + m_CurrentNode = yaml_document_get_root_node(m_ActiveDocument); +} + + +int YAMLRead::YAMLReadCacheHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) +{ + YAMLRead *read = (YAMLRead*)data; + + if (read->m_ReadOffset + size > read->m_EndOffset) + size = read->m_EndOffset - read->m_ReadOffset; + + ReadFileCache (*(CacheReaderBase*)read->m_ReadData, buffer, read->m_ReadOffset, size); + read->m_ReadOffset += size; + *size_read = size; + + return true; +} + +int YAMLRead::YAMLReadStringHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) +{ + YAMLRead *read = (YAMLRead*)data; + + if (read->m_ReadOffset + size > read->m_EndOffset) + size = read->m_EndOffset - read->m_ReadOffset; + + const char* readData = reinterpret_cast<const char*> (read->m_ReadData); + + memcpy (buffer, readData + read->m_ReadOffset, size); + read->m_ReadOffset += size; + *size_read = size; + + return true; +} + +YAMLRead::YAMLRead (const char* strBuffer, int size, int flags, std::string *debugFileName, int debugLineCount) +{ + m_ReadOffset = 0; + m_EndOffset = size; + m_ReadData = const_cast<char*> (strBuffer); + + Init (flags, YAMLReadStringHandler, debugFileName, debugLineCount); +} + +YAMLRead::YAMLRead (const CacheReaderBase *input, size_t readOffset, size_t endOffset, int flags, std::string *debugFileName, int debugLineCount) +{ + m_ReadOffset = readOffset; + m_EndOffset = endOffset; + m_ReadData = (void*)input; + + Init (flags, YAMLReadCacheHandler, debugFileName, debugLineCount); +} + +YAMLRead::~YAMLRead() +{ + yaml_document_delete(&m_Document); +} + +YAMLNode* YAMLRead::GetCurrentNode () +{ + return YAMLDocNodeToNode(m_ActiveDocument, m_CurrentNode); +} + +YAMLNode* YAMLRead::GetValueNodeForKey (const char* key) +{ + return YAMLDocNodeToNode (m_ActiveDocument, GetValueForKey (m_CurrentNode, key)); +} + +int YAMLRead::StringOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + string* theString = reinterpret_cast<string*> (data); + theString->append( (char *) buffer, size); + return 1; +} + +void YAMLRead::BeginMetaGroup (std::string name) +{ + m_MetaParents.push_back(m_CurrentNode); + m_CurrentNode = GetValueForKey(m_CurrentNode, name.c_str()); +} + +void YAMLRead::EndMetaGroup () +{ + m_CurrentNode = m_MetaParents.back(); + m_MetaParents.pop_back(); +} + +void YAMLRead::TransferTypelessData (unsigned size, void* data, int metaFlag) +{ + UnityStr dataString; + Transfer(dataString, "_typelessdata", metaFlag); + dataString.resize (size * 2); + HexStringToBytes (&dataString[0], size, data); +} + +bool YAMLRead::HasNode (const char* name) +{ + return GetValueForKey(m_CurrentNode, name) != NULL; +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLRead.h b/Runtime/Serialize/TransferFunctions/YAMLRead.h new file mode 100644 index 0000000..41e6148 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLRead.h @@ -0,0 +1,422 @@ +#ifndef YAMLREAD_H +#define YAMLREAD_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "TransferNameConversions.h" +#include "YAMLSerializeTraits.h" +#include "Editor/Src/Utility/YAMLNode.h" +#include "External/yaml/include/yaml.h" +#include "Runtime/Serialize/FloatStringConversion.h" +#include "Runtime/Utilities/TypeUtilities.h" + +class CacheReaderBase; +struct StreamingInfo; + +class YAMLRead : public TransferBase +{ +private: + + int m_CurrentVersion; + std::string m_CurrentType; + std::string m_NodeName; + bool m_DidReadLastProperty; + + yaml_document_t m_Document; + yaml_document_t* m_ActiveDocument; + std::vector<yaml_node_t*> m_MetaParents; + std::vector<int> m_Versions; + yaml_node_t* m_CurrentNode; + yaml_node_pair_t* m_CachedIndex; + yaml_read_handler_t *m_ReadHandler; + void* m_ReadData; + size_t m_ReadOffset; + size_t m_EndOffset; + + int GetDataVersion (); + yaml_node_t *GetValueForKey(yaml_node_t* parentNode, const char* keystr); + + static int YAMLReadCacheHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read); + static int YAMLReadStringHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read); + + void Init(int flags, yaml_read_handler_t *handler, std::string *debugFileName, int debugLineCount); + +public: + + YAMLRead (const char* strBuffer, int size, int flags, std::string *debugFileName = NULL, int debugLineCount = 0); + YAMLRead (const CacheReaderBase *input, size_t readOffset, size_t endOffset, int flags, std::string *debugFileName = NULL, int debugLineCount = 0); + YAMLRead (yaml_document_t* yamlDocument, int flags); + ~YAMLRead(); + + void SetVersion (int version) { m_CurrentVersion = version; } + bool IsCurrentVersion () { return m_CurrentVersion == GetDataVersion(); } + bool IsOldVersion (int version) { return version == GetDataVersion(); } + bool IsVersionSmallerOrEqual (int version) { return version >= GetDataVersion(); } + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool IsReadingBackwardsCompatible() { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool DirectStringTransfer () { return true; } + bool AssetMetaDataOnly () { return m_Flags & kAssetMetaDataOnly; } + + bool IsSerializingForGameRelease () { return false; } + + bool DidReadLastProperty () { return m_DidReadLastProperty; } + bool DidReadLastPPtrProperty () { return m_DidReadLastProperty; } + + YAMLNode* GetCurrentNode (); + YAMLNode* GetValueNodeForKey (const char* key); + + static int StringOutputHandler(void *data, unsigned char *buffer, size_t size) ; + void BeginMetaGroup (std::string name); + void EndMetaGroup (); + + template<class T> + void Transfer (T& data, const char* name, int metaFlag = 0); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, int metaFlag = 0); + + void TransferTypeless (unsigned* value, const char* name, int metaFlag = 0) + { + Transfer(*value, name, metaFlag); + } + + bool HasNode (const char* name); + + void TransferTypelessData (unsigned size, void* data, int metaFlag = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferStringData (T& data); + + template<class T> + void TransferSTLStyleArray (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleMap (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleSet (T& data, int metaFlag = 0); + + template<class T> + void TransferPair (T& data, int metaFlag = 0, yaml_node_pair_t* pair = NULL); +}; + +template<> +inline void YAMLRead::TransferBasicData<bool> (bool& data) +{ + int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &i); + data = (i == 0); +} + +template<> +inline void YAMLRead::TransferBasicData<char> (char& data) +{ + //scanf on msvc does not support %hhd. read int instead + int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &i); + data = i; +} + +template<> +inline void YAMLRead::TransferBasicData<SInt8> (SInt8& data) +{ + TransferBasicData (reinterpret_cast<char&> (data)); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt8> (UInt8& data) +{ + //scanf on msvc does not support %hhu. read unsigned int instead + unsigned int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%u", &i); + data = i; +} + +template<> +inline void YAMLRead::TransferBasicData<SInt16> (SInt16& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%hd", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt16> (UInt16& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%hu", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<SInt32> (SInt32& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt32> (UInt32& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%u", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<SInt64> (SInt64& data) +{ + // msvc does not like %lld. Just read hex data directly. + Assert (strlen((char*)m_CurrentNode->data.scalar.value) == 16); + HexStringToBytes ((char*)m_CurrentNode->data.scalar.value, sizeof(SInt64), &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt64> (UInt64& data) +{ + // msvc does not like %lld. Just read hex data directly. + Assert (strlen((char*)m_CurrentNode->data.scalar.value) == 16); + HexStringToBytes ((char*)m_CurrentNode->data.scalar.value, sizeof(UInt64), &data); +} + +template<> +inline void YAMLRead::TransferBasicData<double> (double& data) +{ + data = StringToDoubleAccurate((char*)m_CurrentNode->data.scalar.value); +} + +template<> +inline void YAMLRead::TransferBasicData<float> (float& data) +{ + data = StringToFloatAccurate ((char*)m_CurrentNode->data.scalar.value); +} + +template<class T> +inline void YAMLRead::TransferStringData (T& data) +{ + data = (char*)m_CurrentNode->data.scalar.value; +} + +template<class T> +void YAMLRead::Transfer (T& data, const char* _name, int metaFlag) +{ + m_DidReadLastProperty = false; + + if (metaFlag & kIgnoreInMetaFiles) + return; + + std::string name = YAMLSerializeTraits<T>::ParseName(_name, AssetMetaDataOnly()); + + yaml_node_t *parentNode = m_CurrentNode; + m_CurrentNode = GetValueForKey(parentNode, name.c_str()); + if (!m_CurrentNode) + { + const AllowNameConversion::mapped_type* nameConversion = GetAllowedNameConversions (m_CurrentType.c_str(), _name); + if (nameConversion) + { + for (AllowNameConversion::mapped_type::const_iterator i = nameConversion->begin(); i!=nameConversion->end();i++) + { + m_CurrentNode = GetValueForKey(parentNode, *i); + if (m_CurrentNode) + break; + } + } + if (!m_CurrentNode) + { + if (strcmp (_name, "Base") == 0) + { + if (parentNode && parentNode->type == YAML_MAPPING_NODE) + { + if (parentNode->data.mapping.pairs.start != parentNode->data.mapping.pairs.top) + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, parentNode->data.mapping.pairs.start->value); + } + } + else if (metaFlag & kTransferAsArrayEntryNameInMetaFiles) + { + YAMLSerializeTraits<T>::TransferStringToData (data, m_NodeName); + m_CurrentNode = parentNode; + return; + } + } + } + + std::string parentType = m_CurrentType; + m_CurrentType = SerializeTraits<T>::GetTypeString (&data); + + if (m_CurrentNode != NULL) + { + m_Versions.push_back(-1); + YAMLSerializeTraits<T>::Transfer (data, *this); + m_Versions.pop_back(); + m_DidReadLastProperty = true; + } + + m_CurrentNode = parentNode; + m_CurrentType = parentType; +} + +template<class T> +void YAMLRead::TransferWithTypeString (T& data, const char* name, const char*, int metaFlag) +{ + Transfer(data, name, metaFlag); +} + + +template<class T> +void YAMLRead::TransferSTLStyleArray (T& data, int /*metaFlag*/) +{ + yaml_node_t *parentNode = m_CurrentNode; + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + + switch (m_CurrentNode->type) + { + case YAML_SCALAR_NODE: + { +#if UNITY_BIG_ENDIAN +#error "Needs swapping to be implemented to work on big endian platforms!" +#endif + std::string str; + TransferStringData (str); + size_t byteLength = str.size() / 2; + size_t numElements = byteLength / sizeof(non_const_value_type); + SerializeTraits<T>::ResizeSTLStyleArray (data, numElements); + typename T::iterator dataIterator = data.begin (); + for (size_t i=0; i<numElements; i++) + { + HexStringToBytes (&str[i*2*sizeof(non_const_value_type)], sizeof(non_const_value_type), (void*)&*dataIterator); + ++dataIterator; + } + } + break; + + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t* start = m_CurrentNode->data.sequence.items.start; + yaml_node_item_t* top = m_CurrentNode->data.sequence.items.top; + + SerializeTraits<T>::ResizeSTLStyleArray (data, top - start); + typename T::iterator dataIterator = data.begin (); + + for(yaml_node_item_t* i = start; i != top; i++) + { + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, *i); + YAMLSerializeTraits<non_const_value_type>::Transfer (*dataIterator, *this); + ++dataIterator; + } + } + break; + + // Some stupid old-style meta data writing code unnecessarily used mappings + // instead of sequences to encode arrays. So, we're able to read that as well. + case YAML_MAPPING_NODE: + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + + SerializeTraits<T>::ResizeSTLStyleArray (data, top - start); + typename T::iterator dataIterator = data.begin (); + + for(yaml_node_pair_t* i = start; i != top; i++) + { + yaml_node_t* key = yaml_document_get_node(m_ActiveDocument, i->key); + Assert (key->type == YAML_SCALAR_NODE); + + m_NodeName = (std::string)(char*)key->data.scalar.value; + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, i->value); + + YAMLSerializeTraits<non_const_value_type>::Transfer (*dataIterator, *this); + ++dataIterator; + } + } + break; + + default: + ErrorString("Unexpected node type."); + } + + m_CurrentNode = parentNode; +} + +template<class T> +void YAMLRead::TransferSTLStyleMap (T& data, int metaFlag) +{ + if (m_CurrentNode->type == YAML_MAPPING_NODE) + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + + data.clear(); + + yaml_node_t *parentNode = m_CurrentNode; + + for(yaml_node_pair_t* i = start; i != top; i++) + { + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + typedef typename non_const_value_type::first_type first_type; + non_const_value_type p; + + if (!YAMLSerializeTraits<first_type>::IsBasicType()) + { + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, i->value); + + YAMLSerializeTraits<non_const_value_type>::Transfer (p, *this); + } + else + TransferPair (p, metaFlag, i); + + data.insert (p); + } + m_CurrentNode = parentNode; + } +} + +template<class T> +void YAMLRead::TransferSTLStyleSet (T& data, int /*metaFlag*/) +{ + if (m_CurrentNode->type == YAML_SEQUENCE_NODE) + { + yaml_node_item_t* start = m_CurrentNode->data.sequence.items.start; + yaml_node_item_t* top = m_CurrentNode->data.sequence.items.top; + + data.clear(); + + yaml_node_t *parentNode = m_CurrentNode; + + for(yaml_node_item_t* i = start; i != top; i++) + { + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + non_const_value_type p; + + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, *i); + + YAMLSerializeTraits<non_const_value_type>::Transfer (p, *this); + + data.insert (p); + } + m_CurrentNode = parentNode; + } +} + +template<class T> +void YAMLRead::TransferPair (T& data, int /*metaFlag*/, yaml_node_pair_t* pair) +{ + typedef typename T::first_type first_type; + typedef typename T::second_type second_type; + + if (pair == NULL) + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + if (start == top) + return; + pair = start; + } + + yaml_node_t* parent = m_CurrentNode; + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, pair->key); + YAMLSerializeTraits<first_type>::Transfer (data.first, *this); + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, pair->value); + YAMLSerializeTraits<second_type>::Transfer (data.second, *this); + m_CurrentNode = parent; +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp new file mode 100644 index 0000000..4dfffb8 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp @@ -0,0 +1,139 @@ +#include "UnityPrefix.h" +#include "TransferNameConversions.h" +#include "YAMLRead.h" +#include "YAMLWrite.h" +#include "Runtime/BaseClasses/BaseObject.h" +#include "Editor/Src/GUIDPersistentManager.h" +#include "Runtime/Serialize/SerializedFile.h" +/* +@TODO: + +- Meta file for texture importer has additional unnecessary settings: + TextureImporter: + fileIDToRecycleName: {} + +- Add line offsets to SerializedFile ObjectInfo, so we can have proper line numbers in YAML errors. + +*/ + +// Text transfer of Unity References: +// If NeedsInstanceIDRemapping() is false or for null references: +// {instanceID: id} +// For local objects in same file: +// {fileID: id} +// For objects from other file with GUID: +// {fileID: id, guid: g, type: t} + +template<class T> +void TransferYAMLPtr (T& data, YAMLRead& transfer) +{ + SInt32 instanceID = 0; + if (!transfer.NeedsInstanceIDRemapping()) + { + transfer.Transfer (instanceID, "instanceID"); + data.SetInstanceID (instanceID); + } + else + { + bool allowLocalIdentifier = (transfer.GetFlags () & kYamlGlobalPPtrReference) == 0; + + LocalIdentifierInFileType fileID = 0; + TRANSFER (fileID); + + if (transfer.HasNode("guid")) + { + FileIdentifier id; + transfer.Transfer (id.guid, "guid"); + transfer.Transfer (id.type, "type"); + + id.Fix_3_5_BackwardsCompatibility (); + PersistentManager& pm = GetPersistentManager(); + SInt32 globalIndex = pm.InsertFileIdentifierInternal(id, true); + SerializedObjectIdentifier identifier (globalIndex, fileID); + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + pm.ApplyInstanceIDRemap (identifier); + #endif + + instanceID = pm.SerializedObjectIdentifierToInstanceID (identifier); + } + else if (allowLocalIdentifier) + { + // local fileID + LocalSerializedObjectIdentifier identifier; + identifier.localIdentifierInFile = fileID; + identifier.localSerializedFileIndex = 0; + LocalSerializedObjectIdentifierToInstanceID (identifier, instanceID); + } + + data.SetInstanceID (instanceID); + } +} + +template<class T> +void TransferYAMLPtr (T& data, YAMLWrite& transfer) +{ + transfer.AddMetaFlag(kTransferUsingFlowMappingStyle); + SInt32 instanceID = data.GetInstanceID(); + if (!transfer.NeedsInstanceIDRemapping()) + transfer.Transfer (instanceID, "instanceID"); + else + { + // By default we allow writing self references that exclude guid & type. + // This way references inside of the file will never be lost even if the guid of the file changes + bool allowLocalIdentifier = (transfer.GetFlags () & kYamlGlobalPPtrReference) == 0; + if (allowLocalIdentifier) + { + LocalSerializedObjectIdentifier localIdentifier; + InstanceIDToLocalSerializedObjectIdentifier (instanceID, localIdentifier); + if (localIdentifier.localSerializedFileIndex == 0) + { + transfer.Transfer (localIdentifier.localIdentifierInFile, "fileID"); + return; + } + } + + GUIDPersistentManager& pm = GetGUIDPersistentManager(); + pm.Lock(); + SerializedObjectIdentifier identifier; + if (pm.InstanceIDToSerializedObjectIdentifier(instanceID, identifier)) + { + FileIdentifier id = pm.PathIDToFileIdentifierInternal(identifier.serializedFileIndex); + transfer.Transfer (identifier.localIdentifierInFile, "fileID"); + transfer.Transfer (id.guid, "guid"); + transfer.Transfer (id.type, "type"); + } + else + { + instanceID = 0; + transfer.Transfer (instanceID, "instanceID"); + } + pm.Unlock(); + } +} + +template<> +void TransferYAMLPPtr (PPtr<Object> &data, YAMLRead& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (PPtr<Object> &data, YAMLWrite& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, YAMLRead& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, YAMLWrite& transfer) { TransferYAMLPtr (data, transfer); } + + +template<> +void YAMLSerializeTraits<UnityGUID>::Transfer (UnityGUID& data, YAMLRead& transfer) +{ + std::string str; + transfer.TransferStringData(str); + data = StringToGUID(str); +} + +template<> +void YAMLSerializeTraits<UnityGUID>::Transfer (UnityGUID& data, YAMLWrite& transfer) +{ + std::string str = GUIDToString(data); + transfer.TransferStringData(str); +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h new file mode 100644 index 0000000..04cf3aa --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h @@ -0,0 +1,236 @@ +#ifndef YAMLSERIALIZETRAITS_H +#define YAMLSERIALIZETRAITS_H + +class Object; +template<class T> +class PPtr; +template<class T> +class ImmediatePtr; +struct UnityGUID; +class YAMLRead; +class YAMLWrite; + +template<class T> +class YAMLSerializeTraitsBase +{ + public: + inline static std::string ParseName (const char* _name, bool stripNames) + { + std::string name = _name; + + if (name == "Base") + name = SerializeTraits<T>::GetTypeString (NULL); + else if (stripNames) + { + if (name.rfind(".") != std::string::npos) + name = name.substr(name.rfind(".") + 1); + if (name.length() >= 3 && name.find("m_") == 0) + name = (char)tolower(name[2]) + name.substr(3); + } + return name; + } + + inline static bool ShouldSerializeArrayAsCompactString () + { + return false; + } + + inline static bool IsBasicType () + { + return false; + } + + template<class TransferFunction> inline + static void Transfer (T& data, TransferFunction& transfer) + { + SerializeTraits<T>::Transfer (data, transfer); + } + + inline static void TransferStringToData (T& /*data*/, std::string& /*str*/) + { + } +}; + +template<class T> +class YAMLSerializeTraits : public YAMLSerializeTraitsBase<T> {}; + +template<class T> +class YAMLSerializeTraitsForBasicType : public YAMLSerializeTraitsBase<T> +{ + public: + inline static bool ShouldSerializeArrayAsCompactString () + { + return true; + } + + inline static bool IsBasicType () + { + return true; + } +}; + +template<> +class YAMLSerializeTraits<UInt16> : public YAMLSerializeTraitsForBasicType<UInt16> {}; + +template<> +class YAMLSerializeTraits<SInt16> : public YAMLSerializeTraitsForBasicType<SInt16> {}; + +template<> +class YAMLSerializeTraits<UInt32> : public YAMLSerializeTraitsForBasicType<UInt32> {}; + +template<> +class YAMLSerializeTraits<SInt32> : public YAMLSerializeTraitsForBasicType<SInt32> {}; + +template<> +class YAMLSerializeTraits<UInt64> : public YAMLSerializeTraitsForBasicType<UInt64> {}; + +template<> +class YAMLSerializeTraits<SInt64> : public YAMLSerializeTraitsForBasicType<SInt64> {}; + +template<> +class YAMLSerializeTraits<UInt8> : public YAMLSerializeTraitsForBasicType<UInt8> {}; + +template<> +class YAMLSerializeTraits<SInt8> : public YAMLSerializeTraitsForBasicType<SInt8> {}; + +template<> +class YAMLSerializeTraits<char> : public YAMLSerializeTraitsForBasicType<char> {}; + +template<> +class YAMLSerializeTraits<bool> : public YAMLSerializeTraitsForBasicType<bool> {}; + +template<> +class YAMLSerializeTraits<UnityStr> : public YAMLSerializeTraitsBase<UnityStr > +{ +public: + + template<class TransferFunction> inline + static void Transfer (UnityStr& data, TransferFunction& transfer) + { + transfer.TransferStringData (data); + } + + inline static void TransferStringToData (UnityStr& data, std::string &str) + { + data = str.c_str(); + } + + inline static bool IsBasicType () + { + return true; + } +}; + +// Do not add this serialization function. All serialized strings should use UnityStr instead of std::string +//template<class Traits, class Allocator> +//class YAMLSerializeTraits<std::basic_string<char,Traits,Allocator> > : public YAMLSerializeTraitsBase<std::basic_string<char,Traits,Allocator> > + +template<class FirstClass, class SecondClass> +class YAMLSerializeTraits<std::pair<FirstClass, SecondClass> > : public YAMLSerializeTraitsBase<std::pair<FirstClass, SecondClass> > +{ + public: + + + template<class TransferFunction> inline + static void Transfer (std::pair<FirstClass, SecondClass>& data, TransferFunction& transfer) + { + if (YAMLSerializeTraits<FirstClass>::IsBasicType()) + transfer.TransferPair (data); + else + { + transfer.Transfer (data.first, "first"); + transfer.Transfer (data.second, "second"); + } + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class YAMLSerializeTraits<std::map<FirstClass, SecondClass, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::map<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::map<FirstClass, SecondClass, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class YAMLSerializeTraits<std::multimap<FirstClass, SecondClass, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::multimap<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::multimap<FirstClass, SecondClass, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + + +template<class T, class Compare, class Allocator> +class YAMLSerializeTraits<std::set<T, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::set<T, Compare, Allocator> > +{ + public: + + typedef std::set<T, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleSet (data); + } +}; + + +template<class TransferFunction> +void TransferYAMLPPtr (PPtr<Object> &data, TransferFunction& transfer); +template<class TransferFunction> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, TransferFunction& transfer); + +template<class T> +class YAMLSerializeTraits<PPtr<T> > : public YAMLSerializeTraitsBase<PPtr<T> > +{ + public: + + template<class TransferFunction> inline + static void Transfer (PPtr<T>& data, TransferFunction& transfer) + { + TransferYAMLPPtr ((PPtr<Object>&)data, transfer); + } +}; + +template<class T> +class YAMLSerializeTraits<ImmediatePtr<T> > : public YAMLSerializeTraitsBase<ImmediatePtr<T> > +{ + public: + + template<class TransferFunction> inline + static void Transfer (ImmediatePtr<T>& data, TransferFunction& transfer) + { + TransferYAMLPPtr ((ImmediatePtr<Object>&)data, transfer); + } +}; + +template<> +class YAMLSerializeTraits<UnityGUID> : public YAMLSerializeTraitsBase<UnityGUID> +{ + public: + + template<class TransferFunction> + static void Transfer (UnityGUID& data, TransferFunction& transfer); + + inline static bool IsBasicType () + { + return true; + } +}; +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp b/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp new file mode 100644 index 0000000..ec2ea68 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp @@ -0,0 +1,183 @@ +#include "UnityPrefix.h" +#include "YAMLWrite.h" +#include "../CacheWrap.h" +#include <string> + +void YAMLWrite::TransferStringToCurrentNode (const char* str) +{ + if (m_Error) + return; + int node = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)str, strlen(str), YAML_ANY_SCALAR_STYLE); + if (node) + m_CurrentNode = node; + else + m_Error = true; +} + +int YAMLWrite::NewMapping () +{ + int node = yaml_document_add_mapping(&m_Document, NULL, (m_MetaFlags.back() & kTransferUsingFlowMappingStyle)? YAML_FLOW_MAPPING_STYLE : YAML_ANY_MAPPING_STYLE); + if (node == 0) + m_Error = true; + return node; +} + +int YAMLWrite::NewSequence () +{ + int node = yaml_document_add_sequence(&m_Document, NULL, YAML_ANY_SEQUENCE_STYLE); + if (node == 0) + m_Error = true; + return node; +} + +int YAMLWrite::GetNode () +{ + if (m_CurrentNode == -1) + m_CurrentNode = NewMapping(); + return m_CurrentNode; +} + +void YAMLWrite::AppendToNode(int parentNode, const char* keyStr, int valueNode) +{ + yaml_node_t* parent = yaml_document_get_node(&m_Document, parentNode); + switch (parent->type) + { + case YAML_MAPPING_NODE: + { + int keyNode = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)keyStr, strlen(keyStr), YAML_ANY_SCALAR_STYLE); + if (keyNode == 0) + m_Error = true; + yaml_document_append_mapping_pair(&m_Document, parentNode, keyNode, valueNode); + } + break; + + case YAML_SEQUENCE_NODE: + yaml_document_append_sequence_item(&m_Document, parentNode, valueNode); + break; + + default: + ErrorString("Unexpected node type."); + } +} + +int YAMLWrite::StringOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + string* theString = reinterpret_cast<string*> (data); + theString->append( (char *) buffer, size); + return 1; +} + +int YAMLWrite::CacheOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + CachedWriter* cache = reinterpret_cast<CachedWriter*> (data); + cache->Write(buffer, size); + return 1; +} + +void YAMLWrite::OutputToHandler (yaml_write_handler_t *handler, void *data) +{ + yaml_node_t *root = yaml_document_get_root_node (&m_Document); + if (root->type == YAML_MAPPING_NODE && root->data.mapping.pairs.start != root->data.mapping.pairs.top) + { + yaml_emitter_t emitter; + memset(&emitter, 0, sizeof(emitter)); + + if (!yaml_emitter_initialize (&emitter)) + { + ErrorStringMsg ("Unable to write text file %s: yaml_emitter_initialize failed.", m_DebugFileName.c_str()); + return; + } + + yaml_emitter_set_output(&emitter, handler, data ); + yaml_emitter_dump(&emitter, &m_Document); + + if (emitter.error != YAML_NO_ERROR) + ErrorStringMsg ("Unable to write text file %s: %s.", m_DebugFileName.c_str(), emitter.problem); + + yaml_emitter_delete(&emitter); + } +} + +YAMLWrite::YAMLWrite (int flags, std::string *debugFileName) +{ + if (debugFileName) + m_DebugFileName = *debugFileName; + + memset(&m_Document, 0, sizeof(m_Document)); + + m_CurrentNode = -1; + m_Flags = flags; + m_Error = false; + m_MetaFlags.push_back (0); + m_UserData = NULL; + + if (!yaml_document_initialize(&m_Document, NULL, NULL, NULL, 1, 1)) + { + ErrorStringMsg ("Unable to write text file %s: yaml_document_initialize failed.", m_DebugFileName.c_str()); + m_Error = true; + } +} + +YAMLWrite::~YAMLWrite() +{ + yaml_document_delete(&m_Document); +} + +void YAMLWrite::OutputToCachedWriter (CachedWriter* writer) +{ + if (m_Error) + { + ErrorStringMsg ("Could not serialize text file %s because an error occured - we probably ran out of memory.", m_DebugFileName.c_str()); + return; + } + OutputToHandler (CacheOutputHandler, reinterpret_cast<void *>(writer)); +} + +void YAMLWrite::OutputToString (std::string& str) +{ + if (m_Error) + { + ErrorStringMsg ("Could not serialize text file %s because an error occured - we probably ran out of memory.", m_DebugFileName.c_str()); + return; + } + + OutputToHandler (StringOutputHandler, reinterpret_cast<void *>(&str)); +} + +void YAMLWrite::SetVersion (int version) +{ + char valueStr[256]; + snprintf(valueStr, 256, "%d", version); + int value = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)valueStr, strlen(valueStr), YAML_ANY_SCALAR_STYLE); + + AppendToNode (GetNode(), "serializedVersion", value); +} + +void YAMLWrite::BeginMetaGroup (std::string name) +{ + m_MetaParents.push_back (MetaParent()); + m_MetaParents.back().node = GetNode (); + m_MetaParents.back().name = name; + m_CurrentNode = NewMapping (); +} + +void YAMLWrite::EndMetaGroup () +{ + AppendToNode (m_MetaParents.back().node, m_MetaParents.back().name.c_str(), m_CurrentNode); + m_CurrentNode = m_MetaParents.back().node; + m_MetaParents.pop_back(); +} + +void YAMLWrite::StartSequence () +{ + m_CurrentNode = NewSequence(); +} + +void YAMLWrite::TransferTypelessData (unsigned size, void* data, int metaFlag) +{ + UnityStr dataString; + dataString.resize (size * 2); + BytesToHexString (data, size, &dataString[0]); + Transfer(dataString, "_typelessdata", metaFlag); +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLWrite.h b/Runtime/Serialize/TransferFunctions/YAMLWrite.h new file mode 100644 index 0000000..89c509c --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWrite.h @@ -0,0 +1,340 @@ +#ifndef YAMLWRITE_H +#define YAMLWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "YAMLSerializeTraits.h" +#include "Editor/Src/Utility/YAMLNode.h" +#include "External/yaml/include/yaml.h" +#include "Runtime/Serialize/FloatStringConversion.h" + +class CachedWriter; +struct YAMLConverterContext; + +class YAMLWrite : public TransferBase +{ +private: + + struct MetaParent + { + int node; + std::string name; + }; + + std::vector<MetaParent> m_MetaParents; + std::vector<int> m_MetaFlags; + yaml_document_t m_Document; + int m_CurrentNode; + bool m_Error; + std::string m_DebugFileName; + + void TransferStringToCurrentNode (const char* str); + int NewMapping (); + int NewSequence (); + int GetNode (); + void AppendToNode(int parentNode, const char* keyStr, int valueNode); + static int StringOutputHandler(void *data, unsigned char *buffer, size_t size); + static int CacheOutputHandler(void *data, unsigned char *buffer, size_t size); + + void OutputToHandler (yaml_write_handler_t *handler, void *data); + +public: + + YAMLWrite (int flags, std::string *debugFileName = NULL); + ~YAMLWrite(); + + void OutputToCachedWriter (CachedWriter* writer); + void OutputToString (std::string& str); + + // Sets the "version of the class currently transferred" + void SetVersion (int version); + + bool HasError () { return m_Error; } + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool AssetMetaDataOnly () { return m_Flags & kAssetMetaDataOnly; } + bool IsSerializingForGameRelease () { return false; } + + void SetFlowMappingStyle (bool on); + + yaml_document_t* GetDocument () { return &m_Document; } + int GetCurrentNodeIndex () { return m_CurrentNode; } + + void PushMetaFlag (int flag) { m_MetaFlags.push_back(flag | m_MetaFlags.back());} + void PopMetaFlag () { m_MetaFlags.pop_back(); } + void AddMetaFlag(int mask) { m_MetaFlags.back() |= mask; } + + void BeginMetaGroup (std::string name); + void EndMetaGroup (); + void StartSequence (); + + template<class T> + void Transfer (T& data, const char* name, int metaFlag = 0); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, int metaFlag = 0); + + void TransferTypeless (unsigned* value, const char* name, int metaFlag = 0) + { + Transfer(*value, name, metaFlag); + } + + void TransferTypelessData (unsigned size, void* data, int metaFlag = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferStringData (T& data); + + template<class T> + void TransferSTLStyleArray (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleMap (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleSet (T& data, int metaFlag = 0); + + template<class T> + void TransferPair (T& data, int metaFlag = 0, int parent = -1); +}; + +template<> +inline void YAMLWrite::TransferBasicData<SInt64> (SInt64& data) +{ + char valueStr[17]; + BytesToHexString (&data, sizeof(SInt64), valueStr); + valueStr[16] = '\0'; + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt64> (UInt64& data) +{ + char valueStr[17]; + BytesToHexString (&data, sizeof(UInt64), valueStr); + valueStr[16] = '\0'; + TransferStringToCurrentNode (valueStr); +} + +// This are the definitions of std::numeric_limits<>::max_digits10, which we cannot use +// because it is only in the C++11 standard. +const int kMaxFloatDigits = std::floor(std::numeric_limits<float>::digits * 3010.0/10000.0 + 2); +const int kMaxDoubleDigits = std::floor(std::numeric_limits<double>::digits * 3010.0/10000.0 + 2); + +template<> +inline void YAMLWrite::TransferBasicData<float> (float& data) +{ + char valueStr[64]; + if (FloatToStringAccurate(data, valueStr, 64)) + TransferStringToCurrentNode (valueStr); + else + TransferStringToCurrentNode ("error"); +} + +template<> +inline void YAMLWrite::TransferBasicData<double> (double& data) +{ + char valueStr[64]; + if (DoubleToStringAccurate (data, valueStr, 64)) + TransferStringToCurrentNode (valueStr); + else + TransferStringToCurrentNode ("error"); +} + +template<> +inline void YAMLWrite::TransferBasicData<char> (char& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt8> (SInt8& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt8> (UInt8& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhu", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt32> (SInt32& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%d", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt32> (UInt32& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%u", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt16> (SInt16& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt16> (UInt16& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hu", data); + TransferStringToCurrentNode (valueStr); +} + +template<class T> +inline void YAMLWrite::TransferStringData (T& data) +{ + TransferStringToCurrentNode (data.c_str()); +} + +template<class T> +void YAMLWrite::Transfer (T& data, const char* _name, int metaFlag) +{ + if (m_Error) + return; + + if (metaFlag & kIgnoreInMetaFiles) + return; + + std::string name = YAMLSerializeTraits<T>::ParseName(_name, AssetMetaDataOnly()); + + PushMetaFlag(0); + + int parent = GetNode(); + m_CurrentNode = -1; + + YAMLSerializeTraits<T>::Transfer (data, *this); + + if (m_CurrentNode != -1) + AppendToNode (parent, name.c_str(), m_CurrentNode); + + PopMetaFlag(); + + m_CurrentNode = parent; +} + +template<class T> +void YAMLWrite::TransferWithTypeString (T& data, const char* name, const char*, int metaFlag) +{ + Transfer(data, name, metaFlag); +} + + +template<class T> +void YAMLWrite::TransferSTLStyleArray (T& data, int metaFlag) +{ + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + if (YAMLSerializeTraits<non_const_value_type>::ShouldSerializeArrayAsCompactString()) + { +#if UNITY_BIG_ENDIAN +#error "Needs swapping to be implemented to work on big endian platforms!" +#endif + std::string str; + size_t numElements = data.size(); + size_t numBytes = numElements * sizeof(non_const_value_type); + str.resize (numBytes*2); + + typename T::iterator dataIterator = data.begin (); + for (size_t i=0; i<numElements; i++) + { + BytesToHexString ((void*)&*dataIterator, sizeof(non_const_value_type), &str[i*2*sizeof(non_const_value_type)]); + ++dataIterator; + } + + TransferStringData (str); + } + else + { + m_CurrentNode = NewSequence (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + while (i != end) + { + Transfer (*i, "data", metaFlag); + ++i; + } + } +} + +template<class T> +void YAMLWrite::TransferSTLStyleMap (T& data, int metaFlag) +{ + m_CurrentNode = NewMapping (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + typedef typename non_const_value_type::first_type first_type; + while (i != end) + { + non_const_value_type& p = (non_const_value_type&)(*i); + if (YAMLSerializeTraits<first_type>::IsBasicType()) + TransferPair (p, metaFlag, m_CurrentNode); + else + Transfer (p, "data", metaFlag); + i++; + } +} + +template<class T> +void YAMLWrite::TransferSTLStyleSet (T& data, int metaFlag) +{ + m_CurrentNode = NewSequence (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + while (i != end) + { + non_const_value_type& p = (non_const_value_type&)(*i); + Transfer (p, "data", metaFlag); + ++i; + } +} + +template<class T> +void YAMLWrite::TransferPair (T& data, int /*metaFlag*/, int parent) +{ + typedef typename T::first_type first_type; + typedef typename T::second_type second_type; + if (parent == -1) + parent = NewMapping (); + + m_CurrentNode = -1; + YAMLSerializeTraits<first_type>::Transfer (data.first, *this); + int key = m_CurrentNode; + + m_CurrentNode = -1; + YAMLSerializeTraits<second_type>::Transfer (data.second, *this); + + yaml_document_append_mapping_pair(&m_Document, parent, key, m_CurrentNode); + + m_CurrentNode = parent; +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp b/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp new file mode 100644 index 0000000..b0596d6 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp @@ -0,0 +1,58 @@ +#include "UnityPrefix.h" + +#ifdef ENABLE_UNIT_TESTS + +#include "YAMLWrite.h" +#include "Runtime/Testing/Testing.h" +#include <map> + +SUITE (YAMLWriteTests) +{ + struct Fixture + { + YAMLWrite instanceUnderTest; + Fixture () + : instanceUnderTest (0) {} + }; + + #define ROOT (yaml_document_get_root_node (instanceUnderTest.GetDocument ())) + #define FIRST_KEY_OF(node) (yaml_document_get_node (instanceUnderTest.GetDocument (), node->data.mapping.pairs.start->key)) + #define FIRST_VALUE_OF(node) (yaml_document_get_node (instanceUnderTest.GetDocument (), node->data.mapping.pairs.start->value)) + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithEmptyMap_ProducesMappingNode) + { + std::map<float, UnityStr> testMap; + instanceUnderTest.TransferSTLStyleMap (testMap); + CHECK (ROOT->type == YAML_MAPPING_NODE); + } + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithComplexKey_WritesDataChild) + { + // Arrange. + std::map<PPtr<Object>, UnityStr> testMap; + testMap[PPtr<Object> ()] = "bar"; + + // Act. + instanceUnderTest.TransferSTLStyleMap (testMap); + + // Assert. + CHECK (FIRST_KEY_OF (ROOT)->type == YAML_SCALAR_NODE); + CHECK (strcmp ((const char*) FIRST_KEY_OF (ROOT)->data.scalar.value, "data") == 0); + } + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithBasicTypeKey_DoesNotWriteDataChild) + { + // Arrange. + std::map<int, UnityStr> testMap; + testMap[1234] = "bar"; + + // Act. + instanceUnderTest.TransferSTLStyleMap (testMap); + + // Assert. + CHECK (FIRST_KEY_OF (ROOT)->type == YAML_SCALAR_NODE); + CHECK (strcmp ((const char*) FIRST_KEY_OF (ROOT)->data.scalar.value, "1234") == 0); + } +} + +#endif // ENABLE_UNIT_TESTS diff --git a/Runtime/Serialize/TransferUtility.cpp b/Runtime/Serialize/TransferUtility.cpp new file mode 100644 index 0000000..e98f6c8 --- /dev/null +++ b/Runtime/Serialize/TransferUtility.cpp @@ -0,0 +1,461 @@ +#include "UnityPrefix.h" +#include "TransferUtility.h" +#include "TypeTree.h" +#include "IterateTypeTree.h" +#include "SwapEndianBytes.h" +#include "FileCache.h" +#include "CacheWrap.h" +#include "Runtime/Serialize/TransferFunctions/SerializeTransfer.h" +#include "Runtime/BaseClasses/BaseObject.h" + +#define DEBUG_PRINT_WALK 0 + +using namespace std; + +#if UNITY_EDITOR + +void CountVariables (const TypeTree& typeTree, UInt8** data, int* variableCount); + +std::string ExtractPPtrClassName (const TypeTree& typeTree) +{ + return ExtractPPtrClassName(typeTree.m_Type); +} + +std::string ExtractMonoPPtrClassName (const TypeTree& typeTree) +{ + return ExtractMonoPPtrClassName(typeTree.m_Type); +} + +std::string ExtractPPtrClassName (const std::string& typeName) +{ + if (typeName.size () >= 6 && typeName.find ("PPtr<") == 0) + { + if (typeName[5] == '$') + return std::string (); + else + { + string className (typeName.begin() + 5, typeName.end() - 1); + return className; + } + } + else + return std::string (); +} + +std::string ExtractMonoPPtrClassName (const std::string& typeName) +{ + if (typeName.size () >= 7 && typeName.find ("PPtr<$") == 0) + { + string className (typeName.begin() + 6, typeName.end() - 1); + return className; + } + else + return std::string (); +} + +// Walk through typetree and data to find the bytePosition and variablePosition. +void WalkTypeTree (const TypeTree& typeTree, const UInt8* data, int* bytePosition) +{ + AssertIf (bytePosition == NULL); + +#if DEBUG_PRINT_WALK + const TypeTree* parent = &typeTree; + while (parent->m_Father) + { + parent = parent->m_Father; + printf_console("\t"); + } + + printf_console("%s (%s) position: %d\n", typeTree.m_Type.c_str(), typeTree.m_Name.c_str(), *bytePosition); +#endif + + AssertIf((typeTree.m_ByteSize != -1 && ((typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0 || typeTree.m_Children.empty())) != (typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0)); + + /// During the 2.1 beta we had a bug that generated kAnyChildUsesAlignBytesFlag incorrectly, + /// this was only available in development builds thus we require the (|| typeTree.m_Children.empty()) + bool hasBasicTypeSize = typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0 || typeTree.m_Children.empty(); + + if (hasBasicTypeSize) + { + AssertIf (typeTree.m_ByteSize == -1); + *bytePosition += typeTree.m_ByteSize; + } + else if (typeTree.m_IsArray) + { + // First child in an array is the size + // Second child is the homogenous type of the array + AssertIf (typeTree.m_Children.front ().m_Type != SerializeTraits<SInt32>::GetTypeString (NULL)); + AssertIf (typeTree.m_Children.front ().m_Name != "size"); + AssertIf (typeTree.m_Children.size () != 2); + + SInt32 arraySize, i; + + arraySize = *reinterpret_cast<const SInt32*> (&data[*bytePosition]); + +#if DEBUG_PRINT_WALK + printf_console("Array Size %d position: %d\n", arraySize, *bytePosition); +#endif + + *bytePosition += sizeof (arraySize); + + const TypeTree& elementTypeTree = typeTree.m_Children.back (); + + // If the bytesize is known we can simply skip the recursive loop + if (elementTypeTree.m_ByteSize != -1 && (elementTypeTree.m_MetaFlag & (kAlignBytesFlag | kAnyChildUsesAlignBytesFlag)) == 0) + *bytePosition += arraySize * elementTypeTree.m_ByteSize; + // Otherwise recursively Walk element typetree + else + { + for (i=0;i<arraySize;i++) + WalkTypeTree (elementTypeTree, data, bytePosition); + } + } + else + { + AssertIf (typeTree.m_Children.empty ()); + + TypeTree::TypeTreeList::const_iterator i; + for (i = typeTree.m_Children.begin (); i != typeTree.m_Children.end ();++i) + WalkTypeTree (*i, data, bytePosition); + } + + if (typeTree.m_MetaFlag & kAlignBytesFlag) + { +#if DEBUG_PRINT_WALK + printf_console("Align %d %d", *bytePosition, Align4(*bytePosition)); +#endif + *bytePosition = Align4(*bytePosition); + } +} + +struct ByteSwapGenericIterator +{ + bool operator () (const TypeTree& typeTree, dynamic_array<UInt8>& data, int bytePosition) + { + if (typeTree.IsBasicDataType ()) + { + if (typeTree.m_ByteSize == 4) + { + // Color does not get byteswapped + if (typeTree.m_Father->m_Type != "ColorRGBA") + SwapEndianBytes(*reinterpret_cast<UInt32*> (&data[bytePosition])); + } + else if (typeTree.m_ByteSize == 2) + SwapEndianBytes(*reinterpret_cast<UInt16*> (&data[bytePosition])); + else if (typeTree.m_ByteSize == 8) + SwapEndianBytes(*reinterpret_cast<double*> (&data[bytePosition])); + else if (typeTree.m_ByteSize != 1) + { + AssertString (Format("Unsupported data type when byteswapping %s", typeTree.m_Type.c_str())); + } + + if (typeTree.m_Type == "TypelessData") + { + AssertString ("It is not possible to use Generic byteswap for typeless arrays!"); + } + } + return true; + } +}; + +void ByteSwapGeneric (const TypeTree& typeTree, dynamic_array<UInt8>& data) +{ + ByteSwapGenericIterator functor; + IterateTypeTree (typeTree, data, functor); +} + +SInt32 CalculateByteSize(const TypeTree& type, const UInt8* data) +{ + int position = 0; + WalkTypeTree (type, data, &position); + return position; +} + +#endif // UNITY_EDITOR + +#if !UNITY_EXTERNAL_TOOL + +int FindTypeTreeSeperator (const char* in) +{ + const char* c = in; + while (*c != '.' && *c != '\0') + c++; + return c - in; +} + +const TypeTree* FindAttributeInTypeTreeNoArrays (const TypeTree& typeTree, const char* path) +{ + int seperator = FindTypeTreeSeperator (path); + // Search all typetree children for a name that is the same as the string path with length seperator + for (TypeTree::const_iterator i=typeTree.begin ();i != typeTree.end ();++i) + { + // Early out if size is not the same + if (i->m_Name.size () != seperator) + continue; + + // continue if the name isn't the same + TypeTreeString::const_iterator n = i->m_Name.begin (); + int j; + for (j=0;j<seperator;j++,n++) + { + if (path[j] != *n) + break; + } + if (j != seperator) + continue; + + // We found the attribute we were searching for + if (path[seperator] == '\0') + return &*i; + // Recursively find in the children + else + return FindAttributeInTypeTreeNoArrays (*i, path + seperator + 1); + } + return NULL; +} + +void GenerateTypeTree (Object& object, TypeTree* typeTree, int options) +{ + AssertIf (typeTree == NULL); + *typeTree = TypeTree (); + ProxyTransfer proxy (*typeTree, options, &object, Object::ClassIDToRTTI (object.GetClassID ())->size); + object.VirtualRedirectTransfer (proxy); +} + +template<bool swapEndian> +static inline void WriteObjectToVector (Object& object, dynamic_array<UInt8>* data, int options) +{ + Assert (data != NULL); + data->clear (); + + MemoryCacheWriter memoryCache (*data); + StreamedBinaryWrite<swapEndian> writeStream; + CachedWriter& writeCache = writeStream.Init (options, BuildTargetSelection::NoTarget()); + + writeCache.InitWrite (memoryCache); + object.VirtualRedirectTransfer (writeStream); + + if (!writeCache.CompleteWriting () || writeCache.GetPosition() != data->size ()) + ErrorString ("Error while writing serialized data."); +} + +void WriteObjectToVector (Object& object, dynamic_array<UInt8>* data, int options) +{ + WriteObjectToVector<false> (object, data, options); +} + +template<bool swapEndian> +void ReadObjectFromVector (Object* object, const dynamic_array<UInt8>& data, int options) +{ + Assert (object != NULL); + + MemoryCacheReader memoryCache (const_cast<dynamic_array<UInt8>&> (data)); + StreamedBinaryRead<swapEndian> readStream; + CachedReader& readCache = readStream.Init (options); + readCache.InitRead (memoryCache, 0, data.size ()); + + object->VirtualRedirectTransfer (readStream); + unsigned position = readCache.End (); + + // we read up that object - no need to call Reset as we constructed it fully + object->HackSetResetWasCalled(); + + if (position > (int) data.size ()) + ErrorString ("Error while reading serialized data."); +} + +void ReadObjectFromVector (Object* object, const dynamic_array<UInt8>& data, int options) +{ + ReadObjectFromVector<false> (object, data, options); +} + +#if SUPPORT_TEXT_SERIALIZATION + +string WriteObjectToString (Object& object, int options) +{ + YAMLWrite write (options); + object.VirtualRedirectTransfer (write); + + string result; + write.OutputToString(result); + + return result; +} + +void ReadObjectFromString (Object* object, string& string, int options) +{ + Assert (object != NULL); + YAMLRead read (string.c_str (), string.length (), options); + object->VirtualRedirectTransfer (read); +} + +#endif // SUPPORT_TEXT_SERIALIZATION + +#endif // !UNITY_EXTERNAL_TOOL + +#if UNITY_EDITOR + +void ReadObjectFromVector (Object* object, const dynamic_array<UInt8>& data, const TypeTree& typeTree, int options) +{ + Assert (object != NULL); + + MemoryCacheReader memoryCache (const_cast<dynamic_array<UInt8>&> (data)); + SafeBinaryRead readStream; + CachedReader& readCache = readStream.Init (typeTree, 0, data.size (), options); + readCache.InitRead (memoryCache, 0, data.size ()); + + object->VirtualRedirectTransfer (readStream); + readCache.End (); + + // we will read up that object - no need to call Reset as we will construct it fully + object->HackSetResetWasCalled(); +} + +void WriteObjectToVectorSwapEndian (Object& object, dynamic_array<UInt8>* data, int options) +{ + WriteObjectToVector<true> (object, data, options); +} + +void CountVariables (const TypeTree& typeTree, UInt8** data, int* variableCount) +{ + if (typeTree.m_IsArray) + { + AssertIf (typeTree.m_Children.size () != 2); + AssertIf (typeTree.m_Children.front ().m_Type != "SInt32"); + + SInt32 arraySize = *reinterpret_cast<SInt32*> (*data); + *data += sizeof (SInt32); + *variableCount += 1; + + for (int i=0;i<arraySize;i++) + CountVariables (typeTree.m_Children.back (), data, variableCount); + } + else if (!typeTree.IsBasicDataType ()) + { + for (TypeTree::const_iterator i=typeTree.m_Children.begin ();i != typeTree.m_Children.end ();i++) + CountVariables (*i, data, variableCount); + } + else + { + *variableCount += 1; + *data += typeTree.m_ByteSize; + } +} + +int CountTypeTreeVariables (const TypeTree& typeTree) +{ + if (typeTree.m_Children.empty ()) + return typeTree.m_Index + 1; + else + return CountTypeTreeVariables (*typeTree.m_Children.rbegin ()); +} + +#endif // UNITY_EDITOR + +#if !UNITY_EXTERNAL_TOOL +class IDCollectorFunctor : public GenerateIDFunctor +{ + set<SInt32>* m_IDs; + +public: + + IDCollectorFunctor (set<SInt32>* ptrs) + { + m_IDs = ptrs; + } + virtual ~IDCollectorFunctor () {} + + + virtual SInt32 GenerateInstanceID (SInt32 oldInstanceID, TransferMetaFlags metaFlag = kNoTransferFlags) + { + // Only strong pptrs can insert new ids + if ((metaFlag & kStrongPPtrMask) == 0) + return oldInstanceID; + + Object* object = PPtr<Object> (oldInstanceID); + if (object == NULL) + return oldInstanceID; + + // Already Inserted? + if (!m_IDs->insert (oldInstanceID).second) + return oldInstanceID; + + RemapPPtrTransfer transferFunction (0, false); + transferFunction.SetGenerateIDFunctor (this); + object->VirtualRedirectTransfer (transferFunction); + + return oldInstanceID; + } +}; + +void CollectPPtrs (Object& object, set<SInt32>* collectedPtrs) +{ + IDCollectorFunctor collectFunctor (collectedPtrs); + collectFunctor.GenerateInstanceID (object.GetInstanceID (), kStrongPPtrMask); +} + +void CollectPPtrs (Object& object, vector<Object*>& collectedObjects) +{ + Assert(collectedObjects.empty()); + set<SInt32> output; + CollectPPtrs (object, &output); + + for (set<SInt32>::iterator i=output.begin();i != output.end();i++) + { + Object* obj = dynamic_instanceID_cast<Object*> (*i); + if (obj) + collectedObjects.push_back(obj); + } + +} + +/// We should use a serialize remapper which clears strong pptrs instead. +void CopySerialized(Object& src, Object& dst) +{ + dynamic_array<UInt8> data(kMemTempAlloc); + + if (src.GetClassID() != dst.GetClassID()) + { + ErrorString("Source and Destination Types do not match"); + return; + } + + if (src.GetNeedsPerObjectTypeTree()) + { + // Verify that the typetree matches, otherwise when the script pointer changes comparing the data will read out of bounds. + TypeTree srcTypeTree; + TypeTree dstTypeTree; + GenerateTypeTree (src, &srcTypeTree, kSerializeForPrefabSystem); + GenerateTypeTree (dst, &dstTypeTree, kSerializeForPrefabSystem); + + if (!IsStreamedBinaryCompatbile(srcTypeTree, dstTypeTree)) + { + ErrorString("Source and Destination Types do not match"); + return; + } + } + + + WriteObjectToVector(src, &data, kSerializeForPrefabSystem); + ReadObjectFromVector(&dst, data, kSerializeForPrefabSystem); + +#if UNITY_EDITOR + dst.CloneAdditionalEditorProperties(src); +#endif + + // we copied that object - no need to call Reset as we constructed it fully + dst.HackSetResetWasCalled(); + + dst.AwakeFromLoad(kInstantiateOrCreateFromCodeAwakeFromLoad); + + dst.SetDirty(); +} + +void CompletedReadObjectFromVector (Object& object) +{ + object.CheckConsistency (); + object.AwakeFromLoad (kDefaultAwakeFromLoad); + object.SetDirty (); +} + +#endif // !UNITY_EXTERNAL_TOOL diff --git a/Runtime/Serialize/TransferUtility.h b/Runtime/Serialize/TransferUtility.h new file mode 100644 index 0000000..6c70448 --- /dev/null +++ b/Runtime/Serialize/TransferUtility.h @@ -0,0 +1,100 @@ +#ifndef TRANSFERUTILITY_H +#define TRANSFERUTILITY_H + +#include "Runtime/Utilities/dynamic_array.h" + +class TypeTree; +class Object; +class dynamic_bitset; + +#if !UNITY_EXTERNAL_TOOL + +/// Generates a TypeTree by serializing object using the ProxyTransfer +void GenerateTypeTree (Object& object, TypeTree* typeTree, int options = 0); + +const TypeTree* FindAttributeInTypeTreeNoArrays (const TypeTree& typeTree, const char* propertyPath); + +/// Serializes an object using to a memory buffer as binary data. +/// +/// @note No type information will be written to the buffer. To read the data, +/// a matching version of the serialization code must be used. +void WriteObjectToVector (Object& object, dynamic_array<UInt8>* data, int options = 0); + +/// Unserializes an object from a memory buffer using binary serialization without type trees +/// (meaning the structure of the serialized data must match exactly with what the runtime +/// classes currently look like). +/// +/// @note Dont forget calling CheckConsistency (), AwakeFromLoad (), SetDirty () after calling this method. +void ReadObjectFromVector (Object* object, const dynamic_array<UInt8>& data, int options = 0); + +#if SUPPORT_TEXT_SERIALIZATION + +/// Serialize the given object as text and return the resulting string. +std::string WriteObjectToString (Object& object, int options = 0); + +/// +void ReadObjectFromString (Object* object, std::string& string, int options = 0); + +#endif // SUPPORT_TEXT_SERIALIZATION + +/// Dont forget calling CheckConsistency (), AwakeFromLoad (), SetDirty () after calling ReadObjectFromVector + +/// Call this after using CompletedReadObjectFromVector +void CompletedReadObjectFromVector (Object& object); + +/// Collects the island of ptrs starting with object. +/// the collectedPtrs set should in most cases be empty on calling since when +/// a ptr is already inserted it will not insert his ptrs to the set +void CollectPPtrs (Object& object, std::set<SInt32>* collectedPtrs); +void CollectPPtrs (Object& object, std::vector<Object*>& collectedObjects); + +/// Copies all values from one object to another! +/// - does call set dirty and awake +/// - no pptr remapping or deep copying is done here! +void CopySerialized(Object& src, Object& dst); + +int FindTypeTreeSeperator (const char* in); + +#endif // !UNITY_EXTERNAL_TOOL + + +#if UNITY_EDITOR + +/// Unserializes raw data and a typeTree of the data. +/// The serialized raw data and typeTree can be serialized in a different format. +/// The serializion code will try to match variables and convert data if possible +/// override can be NULL. If it is defined, only variables that are set to be overridden will be written to the object. +/// Every variables override is determined using override[variableTypeTree.m_Index] +/// where variableTypeTree is some child of a variable in typeTree +void ReadObjectFromVector (Object* object, const dynamic_array<UInt8>& data, const TypeTree& typeTree, int options = 0); + +void WriteObjectToVectorSwapEndian (Object& object, dynamic_array<UInt8>* data, int options = 0); + +// Counts the variables that are contained in a typeTree +int CountTypeTreeVariables (const TypeTree& typeTree); + +/// Walks over a typeTree, data pair moving the byteposition while going along. +/// The start position is data + *bytePosition +/// Takes into account arrays. +void WalkTypeTree (const TypeTree& typeTree, const UInt8* data, int* bytePosition); + +/// CalculateByteSize from type and data +SInt32 CalculateByteSize(const TypeTree& type, const UInt8* data); + +/// Byteswaps a data array with typetree. +/// * This will not work correctly if the data contains typeless data transfer! +void ByteSwapGeneric (const TypeTree& typeTree, dynamic_array<UInt8>& data); + +/// Returns the class name part of the PPtr reference. +/// eg. returns "Transform" for "PPtr<Transform>" +/// For Mono classes it returns "" +std::string ExtractPPtrClassName (const TypeTree& typeTree); +std::string ExtractPPtrClassName (const std::string& typeTree); + +/// Returns the class name of the pptr. +std::string ExtractMonoPPtrClassName (const TypeTree& typeTree); +std::string ExtractMonoPPtrClassName (const std::string& typeName); + +#endif // UNITY_EDITOR + +#endif diff --git a/Runtime/Serialize/TypeTree.cpp b/Runtime/Serialize/TypeTree.cpp new file mode 100644 index 0000000..d2e6778 --- /dev/null +++ b/Runtime/Serialize/TypeTree.cpp @@ -0,0 +1,531 @@ +#include "UnityPrefix.h" +#include "TypeTree.h" +#include <iostream> +#include "Runtime/Utilities/LogAssert.h" +#include "Runtime/Utilities/Word.h" +#include "SwapEndianBytes.h" +#include "SerializationMetaFlags.h" +#include "SerializeTraits.h" +#include "Configuration/UnityConfigure.h" + +#if UNITY_EDITOR +#include "Editor/Src/AssetPipeline/MdFourGenerator.h" +#endif + +#if ENABLE_SECURITY +#define TEST_LEN(x) if (iterator + sizeof(x) > end) \ +{\ + return false; \ +} +#else +#define TEST_LEN(x) +#endif + +using namespace std; + +static void RecalculateTypeTreeByteSize (TypeTree& typetree, int options); +static void RecalculateTypeTreeByteSize (TypeTree& typetree, int* typePosition, int options); + + +static void DeprecatedConvertUnity43BetaIntegerTypeNames (TypeTreeString& type) +{ + const char* rawText = type.c_str (); + if (rawText[0] != 'S' && rawText[0] != 'U') + return; + + if (strcmp (rawText, "SInt32") == 0) + type = SerializeTraits<SInt32>::GetTypeString (); + else if (strcmp (rawText, "UInt32") == 0) + type = SerializeTraits<UInt32>::GetTypeString (); +} + + +TypeTree::TypeTree () +{ + m_Father = NULL; + m_Index = -1; + m_ByteOffset = -1; + m_IsArray = false; + m_Version = 1; + m_MetaFlag = kNoTransferFlags; + m_DirectPtr = NULL; + m_ByteSize = -1; +} + +TypeTree::TypeTree (const std::string& name, const std::string& type, SInt32 size) +{ + m_Father = NULL; + m_Index = -1; + m_ByteOffset = -1; + m_IsArray = false; + m_Version = 1; + m_MetaFlag = kNoTransferFlags; + m_DirectPtr = NULL; + + m_Type = type; + m_Name = name; + m_ByteSize = size; +} + +void TypeTree::DebugPrint (string& buffer, int level) const +{ + int i; + for (i=0;i<level;i++) + buffer += "\t"; + buffer += m_Name.c_str(); + buffer += " Type:"; + buffer += m_Type.c_str(); + buffer += " ByteSize:" + IntToString (m_ByteSize); + buffer += " MetaFlag:" + IntToString (m_MetaFlag); + if (m_IsArray) + buffer += " IsArray"; + buffer += "\n"; + + TypeTree::TypeTreeList::const_iterator iter; + for (iter = m_Children.begin ();iter != m_Children.end ();++iter) + iter->DebugPrint (buffer, level + 1); +} + +void GetTypePath (const TypeTree* type, std::string& s) +{ + if (type != NULL) + { + if (type->m_Father != NULL) + { + s += " -> "; + GetTypePath (type->m_Father, s); + } + s += type->m_Name.c_str(); + s += '('; + s += type->m_Type.c_str(); + s += ") "; + } +} + +void TypeTree::operator = (const TypeTree& typeTree) +{ + m_Children = typeTree.m_Children; + for (iterator i = begin ();i != end ();i++) + i->m_Father = this; + m_Type = typeTree.m_Type; + m_Name = typeTree.m_Name; + m_ByteSize = typeTree.m_ByteSize; + m_Index = typeTree.m_Index; + m_IsArray = typeTree.m_IsArray; + m_Version = typeTree.m_Version; + m_MetaFlag = typeTree.m_MetaFlag; +} +/* +bool TypeTree::operator == (const TypeTree& t) const +{ + if (m_ByteSize != t.m_ByteSize || m_Name != t.m_Name || m_Type != t.m_Type || m_Version != t.m_Version) + return false; + return m_Children == t.m_Children; +} +*/ + +bool IsStreamedBinaryCompatbile (const TypeTree& lhs, const TypeTree& rhs) +{ + if (lhs.m_ByteSize != rhs.m_ByteSize || lhs.m_Name != rhs.m_Name || lhs.m_Type != rhs.m_Type || lhs.m_Version != rhs.m_Version) + return false; + if ((lhs.m_MetaFlag & kAlignBytesFlag) != (rhs.m_MetaFlag & kAlignBytesFlag)) + return false; + + if (lhs.m_Children.size() != rhs.m_Children.size()) + return false; + + TypeTree::const_iterator i, k; + for (i=lhs.begin(), k=rhs.begin();i != lhs.end();i++,k++) + { + if (!IsStreamedBinaryCompatbile(*i, *k)) + return false; + } + + return true; +} + +bool IsStreamedBinaryCompatbileAndIndices (const TypeTree& lhs, const TypeTree& rhs) +{ + if (lhs.m_ByteSize != rhs.m_ByteSize || lhs.m_Name != rhs.m_Name || lhs.m_Type != rhs.m_Type || lhs.m_Version != rhs.m_Version || lhs.m_Index != rhs.m_Index) + return false; + if ((lhs.m_MetaFlag & kAlignBytesFlag) != (rhs.m_MetaFlag & kAlignBytesFlag)) + return false; + + if (lhs.m_Children.size() != rhs.m_Children.size()) + return false; + + TypeTree::const_iterator i, k; + for (i=lhs.begin(), k=rhs.begin();i != lhs.end();i++,k++) + { + if (!IsStreamedBinaryCompatbileAndIndices(*i, *k)) + return false; + } + + return true; +} + +void AppendTypeTree (TypeTree& typeTree, TypeTree::iterator begin, TypeTree::iterator end) +{ + Assert(typeTree.m_Father == NULL); + typeTree.m_Children.insert (typeTree.end (), begin, end); + RecalculateTypeTreeByteSize( typeTree, 0); +} + +void AppendTypeTree (TypeTree& typeTree, const TypeTree& typeTreeToAdd) +{ + Assert(typeTree.m_Father == NULL); + typeTree.m_Children.push_back(typeTreeToAdd); + RecalculateTypeTreeByteSize( typeTree, 0); +} + +void RemoveFromTypeTree (TypeTree& typeTree, TypeTree::iterator begin, TypeTree::iterator end) +{ + Assert(typeTree.m_Father == NULL); + typeTree.m_Children.erase (begin, end); + RecalculateTypeTreeByteSize( typeTree, 0); +} + +static void RecalculateTypeTreeByteSize (TypeTree& typetree, int* typePosition, int options) +{ + DebugAssertIf (typetree.m_ByteSize == 0 && typetree.m_Type == "Generic Mono"); + + if ((typetree.m_MetaFlag & kDebugPropertyMask) == 0) + { + typetree.m_Index = *typePosition; + (*typePosition)++; + } + else + { + if (options & kIgnoreDebugPropertiesForIndex) + typetree.m_Index = -1; + else + { + typetree.m_Index = *typePosition; + (*typePosition)++; + } + } + + if (typetree.m_Children.empty ()) + { + AssertIf (typetree.m_IsArray); + return; + } + + bool cantDetermineSize = false; + typetree.m_ByteSize = 0; + + TypeTree::TypeTreeList::iterator i; + for (i = typetree.m_Children.begin ();i != typetree.m_Children.end ();++i) + { + RecalculateTypeTreeByteSize (*i, typePosition, options); + if (i->m_ByteSize == -1) + cantDetermineSize = true; + + if (!cantDetermineSize) + typetree.m_ByteSize += i->m_ByteSize; + + i->m_Father = &typetree; + } + + if (typetree.m_IsArray || cantDetermineSize) + typetree.m_ByteSize = -1; + + DebugAssertIf (typetree.m_ByteSize == 0 && typetree.m_Type == "Generic Mono"); +} + +static void RecalculateTypeTreeByteSize (TypeTree& typetree, int options) +{ +// AssertIf (typetree.m_Father != NULL); +// AssertIf (typetree.m_Name != "Base"); + typetree.m_ByteSize = -1; + int typePosition = 0; + RecalculateTypeTreeByteSize (typetree, &typePosition, options); +} + +/* +void GetTypePath (const TypeTree* type, string& s) +{ + if (type != NULL) + { + GetTypePath (type->m_Father, s); + s += "::"; + s += type->m_Name; + s += '('; + s += type->m_Type; + s += ") "; + } +}*/ + + + +Type::Type () + : m_OldType (NULL), + m_NewType (NULL), + m_Equals (false) +{} + +Type::~Type () +{ + UNITY_DELETE(m_OldType, kMemTypeTree); + UNITY_DELETE(m_NewType, kMemTypeTree); +} + +void Type::SetOldType (TypeTree* t) +{ + UNITY_DELETE(m_OldType, kMemTypeTree); + m_OldType = t; + m_Equals = m_OldType != NULL && m_NewType != NULL && IsStreamedBinaryCompatbile(*m_OldType, *m_NewType); +} + +void Type::SetNewType (TypeTree* t) +{ + UNITY_DELETE (m_NewType, kMemTypeTree); + m_NewType = t; + m_Equals = m_OldType != NULL && m_NewType != NULL && IsStreamedBinaryCompatbile(*m_OldType, *m_NewType); +} + +const TypeTree& GetElementTypeFromContainer (const TypeTree& typeTree) +{ + AssertIf (typeTree.m_Children.size () != 1); + AssertIf (typeTree.m_Children.back ().m_Children.size () != 2); + return typeTree.m_Children.back ().m_Children.back (); +} + +SInt32 GetContainerArraySize (const TypeTree& typeTree, void* data) +{ + AssertIf (data == NULL); + AssertIf (typeTree.m_Children.size () != 1); + AssertIf (typeTree.m_Children.back ().m_Children.size () != 2); + AssertIf (!typeTree.m_Children.back ().m_IsArray); + SInt32* casted = reinterpret_cast<SInt32*> (data); + return *casted; +} + +template<bool kSwap> +bool ReadTypeTreeImpl (TypeTree& t, UInt8 const*& iterator, UInt8 const* end, int version) +{ + // Read Type + if (!ReadString (t.m_Type, iterator, end)) + return false; + + // During the 4.3 beta cycle, we had a couple of versions that were using "SInt32" and "UInt32" + // instead of "int" and "unsigned int". Get rid of these type names here. + // + // NOTE: For now, we always do this. Ideally, we only want this for old data. However, ATM we + // don't version TypeTrees independently (they are tied to kCurrentSerializeVersion). TypeTree + // serialization is going to change soonish so we wait with bumping the version until then. + DeprecatedConvertUnity43BetaIntegerTypeNames (t.m_Type); + + // Read Name + if (!ReadString (t.m_Name, iterator, end)) + return false; + + // Read bytesize + TEST_LEN(t.m_ByteSize); + ReadHeaderCache<kSwap> (t.m_ByteSize, iterator); + + // Read variable count + if (version == 2) + { + SInt32 variableCount; + TEST_LEN(variableCount); + ReadHeaderCache<kSwap> (variableCount, iterator); + } + + // Read Typetree position + if (version != 3) + { + TEST_LEN(t.m_Index); + ReadHeaderCache<kSwap> (t.m_Index, iterator); + } + + // Read IsArray + TEST_LEN(t.m_IsArray); + ReadHeaderCache<kSwap> (t.m_IsArray, iterator); + + // Read version + TEST_LEN(t.m_Version); + ReadHeaderCache<kSwap> (t.m_Version, iterator); + + // Read metaflag + if (version != 3) + { + TEST_LEN(t.m_MetaFlag); + ReadHeaderCache<kSwap> ((UInt32&)t.m_MetaFlag, iterator); + } + + // Read Children count + SInt32 childrenCount; + TEST_LEN(childrenCount); + ReadHeaderCache<kSwap> (childrenCount, iterator); + + enum { kMaxDepth = 50, kMaxChildrenCount = 5000 }; + static int depth = 0; + depth++; + if (depth > kMaxDepth || childrenCount < 0 || childrenCount > kMaxChildrenCount) + { + depth--; + ErrorString ("Fatal error while reading file. Header is invalid!"); + return false; + } + // Read children + for (int i=0;i<childrenCount;i++) + { + TypeTree newType; + t.m_Children.push_back (newType); + if (!ReadTypeTree (t.m_Children.back (), iterator, end, version, kSwap)) + { + depth--; + return false; + } + t.m_Children.back ().m_Father = &t; + } + depth--; + return true; +} + +template<bool kSwap> +void WriteTypeTreeImpl (TypeTree& t, SerializationCache& cache) +{ + // Write type + WriteString (t.m_Type, cache); + + // Write name + WriteString (t.m_Name, cache); + + // Write bytesize + WriteHeaderCache<kSwap> (t.m_ByteSize, cache); + + // Write typetree position + WriteHeaderCache<kSwap> (t.m_Index, cache); + + // Write IsArray + WriteHeaderCache<kSwap> (t.m_IsArray, cache); + + // Write version + WriteHeaderCache<kSwap> (t.m_Version, cache); + + // Write metaflag + WriteHeaderCache<kSwap> ((UInt32)t.m_MetaFlag, cache); + + // Write Children Count + SInt32 childrenCount = t.m_Children.size (); + WriteHeaderCache<kSwap> (childrenCount, cache); + + // Write children + TypeTree::TypeTreeList::iterator i; + for (i = t.m_Children.begin ();i != t.m_Children.end ();i++) + WriteTypeTreeImpl<kSwap> (*i, cache); +} + +void WriteTypeTree (TypeTree& t, SerializationCache& cache, bool swapEndian) +{ + if (swapEndian) + WriteTypeTreeImpl<true> (t, cache); + else + WriteTypeTreeImpl<false> (t, cache); +} + +bool ReadTypeTree (TypeTree& t, UInt8 const*& iterator, UInt8 const* end, int version, bool swapEndian) +{ + if (swapEndian) + return ReadTypeTreeImpl<true> (t, iterator, end, version); + else + return ReadTypeTreeImpl<false> (t, iterator, end, version); +} + +bool ReadVersionedTypeTreeFromVector (TypeTree* typeTree, UInt8 const*& iterator, UInt8 const* end, bool swapEndian) +{ + if (iterator == end) + { + *typeTree = TypeTree (); + return false; + } + SInt32 version; + + if (swapEndian) + { + ReadHeaderCache<true> (version, iterator); + ReadTypeTreeImpl<true> (*typeTree, iterator, end, version); + } + else + { + ReadHeaderCache<false> (version, iterator); + ReadTypeTreeImpl<false> (*typeTree, iterator, end, version); + } + return true; +} + +void WriteString (UnityStr const& s, SerializationCache& cache) +{ + int size = cache.size (); + cache.resize (size + s.size () + 1); + memcpy (&cache[size], s.data (), s.size ()); + cache.back () = '\0'; +} + +void WriteString (TypeTreeString const& s, SerializationCache& cache) +{ + int size = cache.size (); + cache.resize (size + s.size () + 1); + memcpy (&cache[size], s.data (), s.size ()); + cache.back () = '\0'; +} + + +bool ReadString (TypeTreeString& s, UInt8 const*& iterator, UInt8 const* end) +{ + UInt8 const* base = iterator; + while (*iterator != 0 && iterator != end) + iterator++; + +#if ENABLE_SECURITY + if (iterator == end) + return false; +#endif + + s.assign(base, iterator); + iterator++; + return true; +} + +bool ReadString (UnityStr& s, UInt8 const*& iterator, UInt8 const* end) +{ + UInt8 const* base = iterator; + while (*iterator != 0 && iterator != end) + iterator++; + +#if ENABLE_SECURITY + if (iterator == end) + return false; +#endif + + s.assign(base, iterator); + iterator++; + return true; +} + + +#if UNITY_EDITOR +void HashTypeTree (MdFourGenerator& gen, TypeTree& typeTree) +{ + gen.Feed (typeTree.m_Type); + gen.Feed (typeTree.m_Name); + gen.Feed (typeTree.m_ByteSize); + gen.Feed (typeTree.m_IsArray); + gen.Feed (typeTree.m_Version); + + for (TypeTree::iterator it = typeTree.begin (), end = typeTree.end (); it != end; ++it) + { + HashTypeTree (gen, *it); + } +} + +UInt32 HashTypeTree (TypeTree& typeTree) +{ + MdFourGenerator gen; + HashTypeTree (gen, typeTree); + MdFour hash = gen.Finish (); + return hash.PackToUInt32 (); +} +#endif diff --git a/Runtime/Serialize/TypeTree.h b/Runtime/Serialize/TypeTree.h new file mode 100644 index 0000000..0fd6e2f --- /dev/null +++ b/Runtime/Serialize/TypeTree.h @@ -0,0 +1,130 @@ +#ifndef TYPETREE_H +#define TYPETREE_H + +#include <list> +#include <vector> +#include <string> + +#include "Runtime/Misc/Allocator.h" +#include "Configuration/UnityConfigure.h" +#include "SerializationMetaFlags.h" +#include "Runtime/Allocator/MemoryMacros.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + +typedef UNITY_VECTOR(kMemSerialization, UInt8) SerializationCache; + +template<bool kSwap, class T> +// Need this in order for Asset bundles to load correctly when the code is optimized +#if UNITY_BB10 +__attribute__ ((noinline)) +#endif +void ReadHeaderCache (T& t, UInt8 const*& c) +{ + t = *(T const*)c; + if (kSwap) + SwapEndianBytes (t); + c += sizeof (T); +} + +template<bool kSwap, class T> +void WriteHeaderCache (const T& t, SerializationCache& vec) +{ + vec.resize (vec.size () + sizeof (T)); + T& dst = *reinterpret_cast<T*> (&vec[vec.size () - sizeof (T)]); + dst = t; + if (kSwap) + SwapEndianBytes (dst); +} + +UNITY_STR_IMPL(TypeTreeString, kMemTypeTree); + +/// A TypeTree contains information on serialized data. +/// It is a tree storing the name, and type and other information generated by the serialization code, +/// of every variable and all its variables that it might contain. +/// A TypeTree can be generated using a ProxyTransfer class. +/// There are convenience function which generate them from an object in TransferUtility.h + +class TypeTree +{ + public: + + typedef UNITY_LIST(kMemTypeTree,TypeTree) TypeTreeList; + typedef TypeTreeList::iterator iterator; + typedef TypeTreeList::const_iterator const_iterator; + TypeTreeList m_Children; // The children of the Type (eg. a Vector3f has 3 children, float x, float y, float z) + TypeTree* m_Father; + + TypeTreeString m_Type;// The type of the variable (eg. "Vector3f", "int") + TypeTreeString m_Name;// The name of the property (eg. "m_LocalPosition") + SInt32 m_ByteSize;//= -1 if its not determinable (arrays) + SInt32 m_Index; // The index of the property (Prefabs use this index in the override bitset) + SInt32 m_IsArray;// Is the TypeTree an array (first child is the size, second child is the type of the array elements) + SInt32 m_Version; // The version of the serialization format as represented by this type tree. Usually determined by Transfer() functions. + + // Serialization meta data (eg. to hide variables in the property editor) + // Children or their meta flags with their parents! + TransferMetaFlags m_MetaFlag; + SInt32 m_ByteOffset; // The byteoffset into the property in memory. When a variable on the stack is serialized m_ByteOffset is -1. + void* m_DirectPtr; // The direct ptr into the property in memory. NULL if it can't be used. + + TypeTree (); + TypeTree (const std::string& name, const std::string& type, SInt32 size); + + void DebugPrint (std::string& buffer, int level = 0) const; + bool IsBasicDataType ()const { return m_Children.empty () && m_ByteSize > 0; } + + iterator begin () { return m_Children.begin (); } + iterator end () { return m_Children.end (); } + const_iterator begin () const { return m_Children.begin (); } + const_iterator end () const { return m_Children.end (); } + + void operator = (const TypeTree& typeTree); +}; + +void GetTypePath (const TypeTree* type, std::string& s); +const TypeTree& GetElementTypeFromContainer (const TypeTree& typeTree); +SInt32 GetContainerArraySize (const TypeTree& typeTree, void* data); +void AppendTypeTree (TypeTree& typeTree, TypeTree::iterator begin, TypeTree::iterator end); +void AppendTypeTree (TypeTree& typeTree, const TypeTree& typeTreeToAdd); +void RemoveFromTypeTree (TypeTree& typeTree, TypeTree::iterator begin, TypeTree::iterator end); + +class Type +{ + TypeTree* m_OldType; // Type loaded from Disk + TypeTree* m_NewType; // Active type + bool m_Equals; // Are oldType and newType equal + + public: + + Type (); + + ~Type (); + + TypeTree* GetOldType () { return m_OldType; } + TypeTree* GetNewType () { return m_NewType; } + + bool EqualTypes () { return m_Equals; } + + void SetOldType (TypeTree* t); + void SetNewType (TypeTree* t); +}; + +/// Reads/Writes a typetree to a vector<UInt8> with the first four bits being the version of the typetree +bool ReadVersionedTypeTreeFromVector (TypeTree* typeTree, UInt8 const*& iterator, UInt8 const* end, bool swapEndianess); + +/// Reads/Writes a typeTree to a cache, used direcly by SerializedFile +bool ReadTypeTree (TypeTree& t, UInt8 const*& iterator, UInt8 const* end, int version, bool swapEndian); +void WriteTypeTree (TypeTree& t, SerializationCache& cache, bool swapEndianess); + +bool IsStreamedBinaryCompatbile (const TypeTree& lhs, const TypeTree& rhs); +bool IsStreamedBinaryCompatbileAndIndices (const TypeTree& lhs, const TypeTree& rhs); + + +void WriteString (TypeTreeString const& s, SerializationCache& cache); +void WriteString (UnityStr const& s, SerializationCache& cache); +bool ReadString (TypeTreeString& s, UInt8 const*& iterator, UInt8 const* end); +bool ReadString (UnityStr& s, UInt8 const*& iterator, UInt8 const* end); + +UInt32 HashTypeTree (TypeTree& typeTree); + +#endif diff --git a/Runtime/Serialize/WriteData.h b/Runtime/Serialize/WriteData.h new file mode 100644 index 0000000..52be58c --- /dev/null +++ b/Runtime/Serialize/WriteData.h @@ -0,0 +1,27 @@ +#ifndef WRITE_DATA_H +#define WRITE_DATA_H + +#include "SerializationMetaFlags.h" + +struct WriteData +{ + LocalIdentifierInFileType localIdentifierInFile; + SInt32 instanceID; + BuildUsageTag buildUsage; + + WriteData () : localIdentifierInFile(0), instanceID(0) { } + + WriteData (LocalIdentifierInFileType local, SInt32 instance, const BuildUsageTag& tag) + : localIdentifierInFile (local), instanceID(instance), buildUsage(tag) + { } + + WriteData (LocalIdentifierInFileType local, SInt32 instance) + : localIdentifierInFile (local), instanceID(instance) + { } + + friend bool operator < (const WriteData& lhs, const WriteData& rhs) + { + return lhs.localIdentifierInFile < rhs.localIdentifierInFile; + } +}; +#endif diff --git a/Runtime/Serialize/WriteTypeToBuffer.h b/Runtime/Serialize/WriteTypeToBuffer.h new file mode 100644 index 0000000..3065f23 --- /dev/null +++ b/Runtime/Serialize/WriteTypeToBuffer.h @@ -0,0 +1,21 @@ +#ifndef WRITE_TYPE_TO_BUFFER_H +#define WRITE_TYPE_TO_BUFFER_H + +#include "Runtime/Allocator/MemoryManager.h" +#include "Runtime/Utilities/dynamic_array.h" +#include "Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h" + +template <typename TYPE> void WriteTypeToVector (TYPE& object, dynamic_array<UInt8>* data, int options = 0) +{ + data->clear (); + MemoryCacheWriter memoryCache (*data); + StreamedBinaryWrite<false> writeStream; + CachedWriter& writeCache = writeStream.Init (0, BuildTargetSelection::NoTarget()); + + writeCache.InitWrite(memoryCache); + writeStream.Transfer(object, "Base"); + + if (!writeCache.CompleteWriting () || writeCache.GetPosition() != data->size ()) + ErrorString ("Error while writing serialized data."); +} +#endif |