diff options
Diffstat (limited to 'Runtime/Serialize/TransferFunctions')
23 files changed, 4098 insertions, 0 deletions
diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp b/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp new file mode 100644 index 0000000..de4f731 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransfer.cpp @@ -0,0 +1,250 @@ +#include "UnityPrefix.h" +#include "ProxyTransfer.h" +#include "Runtime/Utilities/Word.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + +using namespace std; + +ProxyTransfer::ProxyTransfer (TypeTree& t, int options, void* objectPtr, int objectSize) + : m_TypeTree (t), + m_ActiveFather (NULL) +{ + m_Flags = options; + + m_ObjectPtr = (char*)objectPtr; + m_ObjectSize = objectSize; + m_Index = 0; + m_SimulatedByteOffset = 0; + m_RequireTypelessData = false; + m_DidErrorAlignment = false; +} + +void ProxyTransfer::BeginArrayTransfer (const char* name, const char* typeString, SInt32& size, TransferMetaFlags metaFlag) +{ + AssertIf (m_ActiveFather->m_IsArray); + BeginTransfer (name, typeString, NULL, metaFlag); + m_ActiveFather->m_IsArray = true; + + // transfer size + Transfer (size, "size"); +} + +void ProxyTransfer::BeginTransfer (const char* name, const char* typeString, char* dataPtr, TransferMetaFlags metaFlag) +{ + #if !UNITY_RELEASE + if (m_RequireTypelessData) + { + AssertString ("TransferTypeless needs to be followed by TransferTypelessData with no other variables in between!"); + } + #endif + + #if DEBUGMODE + // skip this check for debug mode inspector, as we can have interface names from C# in the debug data. + if (!(metaFlag & kDebugPropertyMask) && (strstr (name, ".") != NULL || strstr (name, "Array[") != NULL)) + { + string s = "Illegal serialize property name :"; + GetTypePath (m_ActiveFather, s); + s += name; + s += "\n The name may not contain '.' or Array["; + ErrorString (s); + } + #endif + + TypeTree* typeTree; + // Setup a normal typetree child + if (m_ActiveFather != NULL) + { + // Check for multiple occurences of same name + #if DEBUGMODE + TypeTree::TypeTreeList::iterator i; + for (i=m_ActiveFather->m_Children.begin ();i != m_ActiveFather->m_Children.end ();++i) + { + if (i->m_Name == name) + { + string s = "The same field name is serialized multiple names in the class or it's parent class. This is not supported: "; + GetTypePath (m_ActiveFather, s); + s += name; + ErrorString (s); + } + } + #endif + + m_ActiveFather->m_Children.push_back (TypeTree ()); + // Setup new type + typeTree = &m_ActiveFather->m_Children.back (); + typeTree->m_Father = m_ActiveFather; + typeTree->m_Type = typeString; + typeTree->m_Name = name; + typeTree->m_MetaFlag = metaFlag | m_ActiveFather->m_MetaFlag; + AssertIf(typeTree->m_MetaFlag & kAlignBytesFlag); + typeTree->m_MetaFlag &= ~(kAnyChildUsesAlignBytesFlag); + typeTree->m_ByteSize = 0; + } + // Setup root TypeTree + else + { + m_TypeTree.m_Father = NULL; + m_TypeTree.m_Type = typeString; + m_TypeTree.m_Name = name; + m_TypeTree.m_MetaFlag = metaFlag; + m_TypeTree.m_ByteSize = 0; + typeTree = &m_TypeTree; + } + + // Calculate typetree index + if ((typeTree->m_MetaFlag & kDebugPropertyMask) == 0) + typeTree->m_Index = m_Index++; + else + { + if (m_Flags & kIgnoreDebugPropertiesForIndex) + typeTree->m_Index = -1; + else + typeTree->m_Index = m_Index++; + } + + m_ActiveFather = typeTree; + + int offset = dataPtr - m_ObjectPtr; + if (m_ObjectPtr && dataPtr && offset >= 0 && offset < m_ObjectSize) + m_ActiveFather->m_ByteOffset = offset; + + m_ActiveFather->m_DirectPtr = dataPtr; +} + +void ProxyTransfer::AssertContainsNoPPtr (const TypeTree* typeTree) +{ + AssertIf(typeTree->m_Type.find("PPtr<") == 0); + for (TypeTree::const_iterator i=typeTree->begin();i != typeTree->end();i++) + AssertContainsNoPPtr(&*i); +} + +void ProxyTransfer::AssertOptimizeTransfer (int sizeofSize) +{ + if (m_ActiveFather->IsBasicDataType()) + { + AssertIf(sizeofSize != m_ActiveFather->m_ByteSize); + return; + } + + int size = 0; + int baseOffset = m_ActiveFather->m_ByteOffset; + for (TypeTree::const_iterator i=m_ActiveFather->begin();i != m_ActiveFather->end();i++) + { + AssertOptimizeTransferImpl(*i, baseOffset, &size); + } + + // Assert if serialized size is different from sizeof size. + // - Ignore when serializing for game release. We might be serializing differently in that case. (AnimationCurves) + AssertIf(sizeofSize != size && (m_Flags & kSerializeGameRelease) == 0); +} + +void ProxyTransfer::AssertOptimizeTransferImpl (const TypeTree& typetree, int baseOffset, int* totalSize) +{ + if (typetree.m_ByteOffset != -1) + AssertIf (typetree.m_ByteOffset - baseOffset != *totalSize); + + AssertIf (typetree.m_MetaFlag & kAlignBytesFlag); + + if (typetree.IsBasicDataType()) + { + *totalSize += typetree.m_ByteSize; + return; + } + + for (TypeTree::const_iterator i=typetree.begin();i != typetree.end();i++) + AssertOptimizeTransferImpl(*i, baseOffset, totalSize); +} + +void ProxyTransfer::EndTransfer () +{ + TypeTree* current = m_ActiveFather; + // Add bytesize to parent! + m_ActiveFather = m_ActiveFather->m_Father; + if (m_ActiveFather) + { + if (current->m_ByteSize != -1 && m_ActiveFather->m_ByteSize != -1) + m_ActiveFather->m_ByteSize += current->m_ByteSize; + else + m_ActiveFather->m_ByteSize = -1; + + // Propagate if any child uses alignment up to parents + if (current->m_MetaFlag & kAnyChildUsesAlignBytesFlag) + { + m_ActiveFather->m_MetaFlag |= kAnyChildUsesAlignBytesFlag; + } + + DebugAssertIf (m_ActiveFather->m_ByteSize == 0 && m_ActiveFather->m_Type == "Generic Mono"); + } +} + +void ProxyTransfer::EndArrayTransfer () +{ + m_ActiveFather->m_ByteSize = -1; + EndTransfer (); +} + +void ProxyTransfer::SetVersion (int version) +{ + // You can not set the version twice on the same type. + // Probably an inherited class already calls SetVersion + AssertIf (m_ActiveFather->m_Version != 1); + + m_ActiveFather->m_Version = version; +} + +void ProxyTransfer::TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer (name, "TypelessData", size, metaFlag); + + UInt8 temp; + Transfer (temp, "data", metaFlag); + + m_RequireTypelessData = true; + + EndArrayTransfer (); + + Align (); +} + +void ProxyTransfer::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + m_RequireTypelessData = false; +} + +void ProxyTransfer::Align () +{ + m_SimulatedByteOffset = Align4(m_SimulatedByteOffset); + + if (m_ActiveFather && !m_ActiveFather->m_Children.empty()) + { + m_ActiveFather->m_Children.back().m_MetaFlag |= kAlignBytesFlag; + m_ActiveFather->m_MetaFlag |= kAnyChildUsesAlignBytesFlag; + } + else + { + AssertString("Trying to align type data before anything has been serialized!"); + } +} + +#if UNITY_EDITOR +void ProxyTransfer::LogUnalignedTransfer () +{ + if (m_DidErrorAlignment) + return; + + // For now we only support 4 byte alignment + int size = m_ActiveFather->m_ByteSize; + if (size == 8) + size = 4; + if (m_SimulatedByteOffset % size == 0) + return; + + m_DidErrorAlignment = true; + + string path; + GetTypePath(m_ActiveFather, path); + LogString(Format("Unaligned transfer in '%s' at variable '%s'.\nNext unaligned data path: %s", m_TypeTree.m_Type.c_str(), m_ActiveFather->m_Name.c_str(), path.c_str())); +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransfer.h b/Runtime/Serialize/TransferFunctions/ProxyTransfer.h new file mode 100644 index 0000000..662ad05 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransfer.h @@ -0,0 +1,159 @@ +#ifndef PROXYTRANSFER_H +#define PROXYTRANSFER_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" + +class YAMLNode; +struct StreamingInfo; +struct ReduceCopyData; + +class EXPORT_COREMODULE ProxyTransfer : public TransferBase +{ + TypeTree& m_TypeTree; + TypeTree* m_ActiveFather; + char* m_ObjectPtr; + int m_ObjectSize; + int m_Index; + int m_SimulatedByteOffset; + bool m_DidErrorAlignment; + bool m_RequireTypelessData; + friend class MonoBehaviour; + +public: + + ProxyTransfer (TypeTree& t, int flags, void* objectPtr, int objectSize); + + void SetVersion (int version); + bool IsVersionSmallerOrEqual (int version) { Assert(m_ActiveFather->m_Version > version); return false; } + bool IsOldVersion (int version) { Assert(m_ActiveFather->m_Version > version); return false; } + bool NeedNonCriticalMetaFlags () { return (m_Flags & kDontRequireAllMetaFlags) == 0; } + + void AddMetaFlag (TransferMetaFlags flag) { m_ActiveFather->m_MetaFlag = m_ActiveFather->m_MetaFlag | flag; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + void TransferTypelessData (unsigned, void*, int metaData = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> inline + void TransferSTLStyleArrayWithElement (T& elementType, TransferMetaFlags metaFlag); + + void Align (); + + void BeginTransfer (const char* name, const char* typeString, char* data, TransferMetaFlags metaFlag); + void EndTransfer (); + + bool GetTransferFileInfo(unsigned* /*position*/, const char** /*filePath*/) const { return false; } + +private: + + void LogUnalignedTransfer (); + void AssertContainsNoPPtr (const TypeTree* typeTree); + void AssertOptimizeTransfer (int sizeofSize); + void AssertOptimizeTransferImpl (const TypeTree& typetree, int baseOffset, int* totalSize); + void CheckAlignment (); + + + void BeginArrayTransfer (const char* name, const char* typeString, SInt32& size, TransferMetaFlags metaFlag); + void EndArrayTransfer (); +}; + + +template<class T> inline +void ProxyTransfer::TransferSTLStyleArray (T& /*data*/, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + typename T::value_type p; + Transfer (p, "data"); + + // Make sure MightContainPPtr and AllowTransferOptimization is setup correctly + DebugAssertIf(SerializeTraits<T>::AllowTransferOptimization()); + + EndArrayTransfer (); +} + +template<class T> inline +void ProxyTransfer::TransferSTLStyleArrayWithElement (T& elementType, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + Transfer (elementType, "data"); + + EndArrayTransfer (); +} + + +template<class T> inline +void ProxyTransfer::TransferSTLStyleMap (T&, TransferMetaFlags metaFlag) +{ + SInt32 size; + BeginArrayTransfer ("Array", "Array", size, metaFlag); + + typename NonConstContainerValueType<T>::value_type p; + Transfer (p, "data"); + + #if !UNITY_RELEASE + DebugAssertIf(SerializeTraits<T>::AllowTransferOptimization()); + #endif + + EndArrayTransfer (); +} + +template<class T> inline +void ProxyTransfer::Transfer (T& data, const char* name, TransferMetaFlags metaFlag) +{ + BeginTransfer (name, SerializeTraits<T>::GetTypeString (&data), (char*)&data, metaFlag); + SerializeTraits<T>::Transfer (data, *this); + + // Make sure MightContainPPtr and AllowTransferOptimization is setup correctly + #if !UNITY_RELEASE + if (!SerializeTraits<T>::MightContainPPtr()) + AssertContainsNoPPtr(m_ActiveFather); + if (SerializeTraits<T>::AllowTransferOptimization()) + AssertOptimizeTransfer(SerializeTraits<T>::GetByteSize()); + #endif + + EndTransfer (); +} + +template<class T> inline +void ProxyTransfer::TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag) +{ + BeginTransfer (name, typeName, (char*)&data, metaFlag); + SerializeTraits<T>::Transfer (data, *this); + EndTransfer (); +} + + +template<class T> +inline void ProxyTransfer::TransferBasicData (T&) +{ + m_ActiveFather->m_ByteSize = SerializeTraits<T>::GetByteSize (); + #if UNITY_EDITOR + if (m_SimulatedByteOffset % m_ActiveFather->m_ByteSize != 0) + { + LogUnalignedTransfer(); + } + m_SimulatedByteOffset += m_ActiveFather->m_ByteSize; + #endif +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp b/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp new file mode 100644 index 0000000..2f62542 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/ProxyTransferTests.cpp @@ -0,0 +1,216 @@ +#include "UnityPrefix.h" + +#if ENABLE_UNIT_TESTS +#include "External/UnitTest++/src/UnitTest++.h" +#include "Runtime/Serialize/TypeTree.h" +#include "Runtime/Serialize/TransferFunctions/ProxyTransfer.h" + +struct TypeTreeTestItem +{ +public: + DECLARE_SERIALIZE (TypeTreeTestItem); + + float m_float; + double m_double; + int m_int; // Same as SInt32 to serialization system. + unsigned int m_unsignedint; // Same as UInt32 to serialization system. + SInt64 m_SInt64; + UInt64 m_UInt64; + SInt32 m_SInt32; + UInt32 m_UInt32; + SInt16 m_SInt16; + UInt16 m_UInt16; + SInt8 m_SInt8; + UInt8 m_UInt8; + char m_char; + bool m_bool; + UnityStr m_UnityStr; +}; + +template<class TransferFunction> +void TypeTreeTestItem::Transfer (TransferFunction& transfer) +{ + TRANSFER (m_float); + TRANSFER (m_double); + TRANSFER (m_int); + TRANSFER (m_unsignedint); + TRANSFER (m_SInt64); + TRANSFER (m_UInt64); + TRANSFER (m_SInt32); + TRANSFER (m_UInt32); + TRANSFER (m_SInt16); + TRANSFER (m_UInt16); + TRANSFER (m_SInt8); + TRANSFER (m_UInt8); + TRANSFER (m_char); + TRANSFER (m_bool); + TRANSFER (m_UnityStr); +} + +SUITE (ProxyTransferTests) +{ + TEST (TypeTree_GenerateBaseTypes) + { + // Create test item. + TypeTreeTestItem item; + + // Populate test item. + item.m_float = 123.0f; + item.m_double = 456.0; + item.m_int = 2^31-1; + item.m_unsignedint = 2^32-1; + item.m_SInt64 = 2^63-1; + item.m_UInt64 = 2^64-1; + item.m_SInt32 = 2^31-1; + item.m_UInt32 = 2^32-1; + item.m_SInt16 = 2^15-1; + item.m_UInt16 = 2^16-1; + item.m_SInt8 = 2^7-1; + item.m_UInt8 = 2^8-1; + item.m_char = '@'; + item.m_bool = true; + item.m_UnityStr = "ProxyTransferTests"; + + // Generate type tree. + TypeTree tree; + ProxyTransfer proxy (tree, 0, &item, sizeof(TypeTreeTestItem)); + proxy.Transfer( item, "Base" ); + + // Validate the following expected output: + // + // TypeTree: Base Type:TypeTreeTestItem ByteSize:-1 MetaFlag:32768 + // m_float Type:float ByteSize:4 MetaFlag:0 + // m_double Type:double ByteSize:8 MetaFlag:0 + // m_int Type:int ByteSize:4 MetaFlag:0 + // m_unsignedint Type:unsigned int ByteSize:4 MetaFlag:0 + // m_SInt64 Type:SInt64 ByteSize:8 MetaFlag:0 + // m_UInt64 Type:UInt64 ByteSize:8 MetaFlag:0 + // m_SInt32 Type:SInt32 ByteSize:4 MetaFlag:0 + // m_UInt32 Type:UInt32 ByteSize:4 MetaFlag:0 + // m_SInt16 Type:SInt16 ByteSize:2 MetaFlag:0 + // m_UInt16 Type:UInt16 ByteSize:2 MetaFlag:0 + // m_SInt8 Type:SInt8 ByteSize:1 MetaFlag:0 + // m_UInt8 Type:UInt8 ByteSize:1 MetaFlag:0 + // m_char Type:char ByteSize:1 MetaFlag:0 + // m_bool Type:bool ByteSize:1 MetaFlag:0 + // m_UnityStr Type:string ByteSize:-1 MetaFlag:32768 + // Array Type:Array ByteSize:-1 MetaFlag:16385 IsArray + // size Type:SInt32 ByteSize:4 MetaFlag:1 + // data Type:char ByteSize:1 MetaFlag:1 + + // Check correct number of children. + const int expectedChildrenCount = 15; + CHECK_EQUAL (expectedChildrenCount, tree.m_Children.size()); + + // Transfer children for explicit checking. + TypeTree childTree[expectedChildrenCount]; + int childIndex = 0; + for( TypeTree::iterator treeItr = tree.begin(); treeItr != tree.end(); ++treeItr ) + childTree[childIndex++] = *treeItr; + + // Check children types... + + TypeTree* child = childTree; + CHECK_EQUAL (SerializeTraits<float>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_float", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<double>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_double", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_int", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_unsignedint", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt64>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt64", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt64>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt64", child->m_Name); + CHECK_EQUAL (8, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt32", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt32>::GetTypeString(), child->m_Type); + CHECK_EQUAL ("m_UInt32", child->m_Name); + CHECK_EQUAL (4, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt16>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt16", child->m_Name); + CHECK_EQUAL (2, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt16>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt16", child->m_Name); + CHECK_EQUAL (2, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<SInt8>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_SInt8", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<UInt8>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_UInt8", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL (SerializeTraits<char>::GetTypeString (), child->m_Type); + CHECK_EQUAL ("m_char", child->m_Name); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL ("m_bool", child->m_Name); + CHECK_EQUAL (SerializeTraits<bool>::GetTypeString (), child->m_Type); + CHECK_EQUAL (1, child->m_ByteSize); + CHECK_EQUAL (0, child->m_Children.size()); + + ++child; + CHECK_EQUAL ("m_UnityStr", child->m_Name); // String. + CHECK_EQUAL (SerializeTraits<UnityStr>::GetTypeString (), child->m_Type); + CHECK_EQUAL (-1, child->m_ByteSize); + CHECK_EQUAL (1, child->m_Children.size()); + CHECK_EQUAL ("Array", (*child->begin()).m_Name); // String is an array. + CHECK_EQUAL ("Array", (*child->begin()).m_Type); + CHECK_EQUAL (-1, (*child->begin()).m_ByteSize); + CHECK ( (*child->begin()).m_IsArray != 0); + CHECK_EQUAL ("size", (*(*child->begin()).begin()).m_Name); // Array size. + CHECK_EQUAL (SerializeTraits<SInt32>::GetTypeString (), (*(*child->begin()).begin()).m_Type); + CHECK_EQUAL (4, (*(*child->begin()).begin()).m_ByteSize); + CHECK_EQUAL ("data", (*++(*child->begin()).begin()).m_Name); // Array data. + CHECK_EQUAL (SerializeTraits<char>::GetTypeString (), (*++(*child->begin()).begin()).m_Type); + CHECK_EQUAL (1, (*++(*child->begin()).begin()).m_ByteSize); + } +} + +#endif //ENABLE_UNIT_TESTS + diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp new file mode 100644 index 0000000..62c5a36 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.cpp @@ -0,0 +1,31 @@ +#include "UnityPrefix.h" +#include "RemapPPtrTransfer.h" + +RemapPPtrTransfer::RemapPPtrTransfer (int flags, bool readPPtrs) +{ + m_ReadPPtrs = readPPtrs; + m_Flags = flags; + m_UserData = NULL; + m_GenerateIDFunctor = NULL; + m_MetaMaskStack.reserve(4); + m_MetaMaskStack.push_back (kNoTransferFlags); + m_CachedMetaMaskStackTop = kNoTransferFlags; +} + +void RemapPPtrTransfer::PushMetaFlag (TransferMetaFlags flag) +{ + m_MetaMaskStack.push_back (m_MetaMaskStack.back() | flag); + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +} + +void RemapPPtrTransfer::PopMetaFlag () +{ + m_MetaMaskStack.pop_back(); + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +} + +void RemapPPtrTransfer::AddMetaFlag (TransferMetaFlags flag) +{ + m_MetaMaskStack.back () |= flag; + m_CachedMetaMaskStackTop = m_MetaMaskStack.back (); +}
\ No newline at end of file diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h new file mode 100644 index 0000000..853d83b --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransfer.h @@ -0,0 +1,178 @@ +#ifndef REMAPPPTRTRANSFER_H +#define REMAPPPTRTRANSFER_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" +#include "Runtime/Allocator/MemoryMacros.h" + +#include <stack> + +template<class T> +class PPtr; +template<class T> +class ImmediatePtr; +struct StreamingInfo; + +class GenerateIDFunctor +{ + public: + + /// Calls GenerateInstanceID for every PPtr that is found while transferring. + /// oldInstanceID is the instanceID of the PPtr. metaFlag is the ored metaFlag of the the Transfer trace to the currently transferred pptr. + /// After GenerateInstanceID returns, the PPtr will be set to the returned instanceID + /// If you dont want to change the PPtr return oldInstanceID + virtual SInt32 GenerateInstanceID (SInt32 oldInstanceID, TransferMetaFlags metaFlag) = 0; +}; + + +////@todo: Think about a smart way to calculate to let the compiler optimize transfer code of non-pptr classes away +//// currently we have a serializeTraits::MightContainPPtr but this should be somehow automatic! + + +/// Transfer that scans for PPtrs and optionally allows to replace them in-place. Is given a GenerateIDFunctor which maps one +/// or more existing instance IDs to new instance IDs and then crawls through an object's transfers looking for PPtr transfers. +/// Does not touch any other data. +class EXPORT_COREMODULE RemapPPtrTransfer : public TransferBase +{ +private: + + GenerateIDFunctor* m_GenerateIDFunctor; + UNITY_TEMP_VECTOR(TransferMetaFlags) m_MetaMaskStack; + TransferMetaFlags m_CachedMetaMaskStackTop; + bool m_ReadPPtrs; + +public: + + RemapPPtrTransfer (int flags, bool readPPtrs); + + void SetGenerateIDFunctor (GenerateIDFunctor* functor) { m_GenerateIDFunctor = functor; } + GenerateIDFunctor* GetGenerateIDFunctor () const { return m_GenerateIDFunctor; } + + bool IsReadingPPtr () { return m_ReadPPtrs; } + bool IsWritingPPtr () { return true; } + bool IsRemapPPtrTransfer () { return true; } + + bool DidReadLastPPtrProperty () { return true; } + + void AddMetaFlag (TransferMetaFlags flag); + + void PushMetaFlag (TransferMetaFlags flag); + void PopMetaFlag (); + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned*, const char*, TransferMetaFlags /*metaFlag*/ = kNoTransferFlags) { } + + void TransferTypelessData (unsigned, void*, TransferMetaFlags /*metaFlag*/ = kNoTransferFlags) { } + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + SInt32 GetNewInstanceIDforOldInstanceID (SInt32 oldInstanceID) { return m_GenerateIDFunctor->GenerateInstanceID (oldInstanceID, m_CachedMetaMaskStackTop);} +}; + +template<class T> +struct RemapPPtrTraits +{ + static bool IsPPtr () { return false; } + static int GetInstanceIDFromPPtr (const T& ) { AssertString ("Never"); return 0; } + static void SetInstanceIDOfPPtr (T& , SInt32) { AssertString ("Never"); } +}; + +template<class T> +struct RemapPPtrTraits<PPtr<T> > +{ + static bool IsPPtr () { return true; } + static int GetInstanceIDFromPPtr (const PPtr<T>& data) { return data.GetInstanceID (); } + static void SetInstanceIDOfPPtr (PPtr<T>& data, SInt32 instanceID) { data.SetInstanceID (instanceID); } +}; + +template<class T> +struct RemapPPtrTraits<ImmediatePtr<T> > +{ + static bool IsPPtr () { return true; } + static int GetInstanceIDFromPPtr (const ImmediatePtr<T>& data) { return data.GetInstanceID (); } + static void SetInstanceIDOfPPtr (ImmediatePtr<T>& data, SInt32 instanceID) { data.SetInstanceID (instanceID); } +}; + +template<class T> +void RemapPPtrTransfer::Transfer (T& data, const char*, TransferMetaFlags metaFlag) +{ + if (SerializeTraits<T>::MightContainPPtr ()) + { + if (metaFlag != 0) + PushMetaFlag(metaFlag); + + if (RemapPPtrTraits<T>::IsPPtr ()) + { + AssertIf (m_GenerateIDFunctor == NULL); + ANALYSIS_ASSUME (m_GenerateIDFunctor); + SInt32 oldInstanceID = RemapPPtrTraits<T>::GetInstanceIDFromPPtr (data); + SInt32 newInstanceID = GetNewInstanceIDforOldInstanceID(oldInstanceID); + + if (m_ReadPPtrs) + { + RemapPPtrTraits<T>::SetInstanceIDOfPPtr (data, newInstanceID); + } + else + { + AssertIf(oldInstanceID != newInstanceID); + } + } + else + SerializeTraits<T>::Transfer (data, *this); + + if (metaFlag != 0) + PopMetaFlag(); + } +} + +template<class T> +void RemapPPtrTransfer::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags metaFlag) +{ + Transfer(data, NULL, metaFlag); +} + + +template<class T> +void RemapPPtrTransfer::TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag) +{ + if (SerializeTraits<typename T::value_type>::MightContainPPtr ()) + { + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + while (i != end) + { + Transfer (*i, "data", metaFlag); + ++i; + } + } +} + +template<class T> +void RemapPPtrTransfer::TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag) +{ + typedef typename NonConstContainerValueType<T>::value_type NonConstT; + if (SerializeTraits<NonConstT>::MightContainPPtr ()) + { + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + while (i != end) + { + NonConstT& p = (NonConstT&) (*i); + Transfer (p, "data", metaFlag); + i++; + } + } +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp b/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp new file mode 100644 index 0000000..ea4710b --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/RemapPPtrTransferTests.cpp @@ -0,0 +1,57 @@ +#include "UnityPrefix.h" +#include "Configuration/UnityConfigure.h" + +#if ENABLE_UNIT_TESTS + +#include "Runtime/Testing/Testing.h" +#include "Runtime/Testing/TestFixtures.h" + +SUITE (RemapPPtrTransferTests) +{ + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DoesNotTouchNonPPtrProperty) + { + UnityStr m_NonPPtrProperty = "test"; + TRANSFER (m_NonPPtrProperty); + CHECK (m_NonPPtrProperty == "test"); + } + + TEST_FIXTURE (DoesNotTouchNonPPtrPropertyTestFixture, Transfer_WithNonPPtrProperty_DoesNotChangeProperty) + { + DoRemapPPtrTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (RemapsPPtrProperty) + { + PPtr<Object> m_PPtrProperty (1234); + TRANSFER (m_PPtrProperty); + CHECK (m_PPtrProperty.GetInstanceID () == 4321); + } + + TEST_FIXTURE (RemapsPPtrPropertyTestFixture, Transfer_WithPPtrProperty_MapsToNewInstanceID) + { + AddPPtrRemap (1234, 4321); + DoRemapPPtrTransfer (); + } + + //------------------------------------------------------------------------- + + DEFINE_TRANSFER_TEST_FIXTURE (DidReadLastPPtrProperty) + { + PPtr<Object> m_PPtrProperty; + TRANSFER (m_PPtrProperty); + CHECK (transfer.DidReadLastPPtrProperty ()); + } + + TEST_FIXTURE (DidReadLastPPtrPropertyTestFixture, DidReadLastPPtrProperty_WithPPtrProperty_IsTrue) + { + DoRemapPPtrTransfer (); + } + +} //TEST + +#endif // ENABLE_UNIT_TESTS + diff --git a/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp new file mode 100644 index 0000000..0fe91e8 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.cpp @@ -0,0 +1,487 @@ +#include "UnityPrefix.h" +#include "SafeBinaryRead.h" +#include "TransferNameConversions.h" +#include "Runtime/Utilities/dynamic_bitset.h" +#include "Runtime/Utilities/algorithm_utility.h" +#include "Runtime/Utilities/InitializeAndCleanup.h" + +#define LOG_MISSING_VARIBALES 0 + +#if SUPPORT_SERIALIZED_TYPETREES + +using namespace std; + +typedef map<pair<char*, char*>, ConversionFunction*, smaller_cstring_pair> ConverterFunctions; +static ConverterFunctions* gConverterFunctions; + +namespace SafeBinaryReadManager +{ + void StaticInitialize() + { + gConverterFunctions = UNITY_NEW(ConverterFunctions,kMemSerialization); + } + void StaticDestroy() + { + UNITY_DELETE(gConverterFunctions,kMemSerialization); + } +} +static RegisterRuntimeInitializeAndCleanup s_SafeBinaryReadManagerCallbacks(SafeBinaryReadManager::StaticInitialize, SafeBinaryReadManager::StaticDestroy); + +ConversionFunction* FindConverter (const char* oldType, const char* newTypeName) +{ + pair<char*, char*> arg = make_pair(const_cast<char*> (oldType), const_cast<char*> (newTypeName)); + + ConverterFunctions::iterator found = gConverterFunctions->find (arg); + if (found == gConverterFunctions->end()) + return NULL; + + return found->second; +} + +void SafeBinaryRead::RegisterConverter (const char* oldType, const char* newType, ConversionFunction* converter) +{ + + pair<char*, char*> arg = make_pair(const_cast<char*> (oldType), const_cast<char*> (newType)); + AssertMsg (!gConverterFunctions->count (arg), "Duplicate conversion registered"); + (*gConverterFunctions)[arg] = converter; +} + +void SafeBinaryRead::CleanupConverterTable () +{ + (*gConverterFunctions).clear(); +} + +static void Walk (const TypeTree& typeTree, CachedReader& cache, SInt32* bytePosition, bool endianSwap); + +CachedReader& SafeBinaryRead::Init (const TypeTree& oldBase, int bytePosition, int byteSize, int flags) +{ + AssertIf (!m_StackInfo.empty ()); + m_OldBaseType = &oldBase; + m_BaseBytePosition = bytePosition; + AssertIf (m_BaseBytePosition < 0); + m_BaseByteSize = byteSize; + m_Flags = flags; + m_UserData = NULL; + m_DidReadLastProperty = false; + #if UNITY_EDITOR + m_TypeTreeHasChanged = false; + #endif + return m_Cache; +} + +CachedReader& SafeBinaryRead::Init (SafeBinaryRead& transfer) +{ + int newBasePosition = transfer.m_StackInfo.top ().bytePosition; + int size = transfer.m_BaseByteSize - (newBasePosition - transfer.m_BaseBytePosition); + Init (*transfer.m_StackInfo.top ().type, newBasePosition, size, transfer.m_Flags); + m_Cache.InitRead (*transfer.m_Cache.GetCacher(), transfer.m_StackInfo.top ().bytePosition, size); + m_UserData = NULL; + m_DidReadLastProperty = false; + #if UNITY_EDITOR + m_TypeTreeHasChanged = false; + #endif + + return m_Cache; +} + +SafeBinaryRead::~SafeBinaryRead () +{ + AssertIf (!m_StackInfo.empty ()); + AssertIf (!m_PositionInArray.empty ()); +} + +static void Walk (const TypeTree& typeTree, CachedReader& cache, SInt32* bytePosition, bool endianSwap) +{ + AssertIf (bytePosition == NULL); + + AssertIf((typeTree.m_ByteSize != -1 && ((typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0 || typeTree.m_Children.empty())) != (typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0)); + + if (typeTree.m_ByteSize != -1 && (typeTree.m_MetaFlag & kAnyChildUsesAlignBytesFlag) == 0) + { + *bytePosition += typeTree.m_ByteSize; + } + else if (typeTree.m_IsArray) + { + // First child in an array is the size + // Second child is the homogenous type of the array + AssertIf (typeTree.m_Children.front ().m_Type != SerializeTraits<SInt32>::GetTypeString (NULL)); + AssertIf (typeTree.m_Children.front ().m_Name != "size"); + AssertIf (typeTree.m_Children.size () != 2); + + SInt32 arraySize, i; + cache.Read (arraySize, *bytePosition); + if (endianSwap) + SwapEndianBytes(arraySize); + + *bytePosition += sizeof (arraySize); + + const TypeTree& elementTypeTree = typeTree.m_Children.back (); + + // If the bytesize is known we can simply skip the recursive loop + if (elementTypeTree.m_ByteSize != -1 && (elementTypeTree.m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + *bytePosition += arraySize * elementTypeTree.m_ByteSize; + // Otherwise recursively Walk element typetree + else + { + for (i=0;i<arraySize;i++) + Walk (typeTree.m_Children.back (), cache, bytePosition, endianSwap); + } + } + else + { + TypeTree::TypeTreeList::const_iterator i; + for (i = typeTree.m_Children.begin (); i != typeTree.m_Children.end ();++i) + Walk (*i, cache, bytePosition, endianSwap); + } + + if (typeTree.m_MetaFlag & kAlignBytesFlag) + { + #if UNITY_EDITOR +// const TypeTree* root = &typeTree; +// while (root->m_Father != NULL) +// root = root->m_Father; +// if (root->m_Type == "MonoBehaviour") +// ErrorString("Alignment in monobehaviour???"); + #endif + *bytePosition = Align4(*bytePosition); + } +} + +// Walk through typetree and data to find the bytePosition +void SafeBinaryRead::Walk (const TypeTree& typeTree, SInt32* bytePosition) +{ + ::Walk (typeTree, m_Cache, bytePosition, ConvertEndianess()); +} + +void SafeBinaryRead::OverrideRootTypeName (const char* typeString) +{ + Assert(m_StackInfo.size() == 1); + m_StackInfo.top().currentTypeName = typeString; + #if !UNITY_RELEASE + m_StackInfo.top().currentTypeNameCheck = typeString; + #endif +} + + +int SafeBinaryRead::BeginTransfer (const char* name, const char* typeString, ConversionFunction** converter) +{ + if (converter != NULL) + *converter = NULL; + + m_DidReadLastProperty = false; + + // For the first Transfer only setup the stack to the base parameters + if (m_StackInfo.empty ()) + { + ErrorIf (name != m_OldBaseType->m_Name); + + StackedInfo newInfo; + newInfo.type = m_OldBaseType; + newInfo.bytePosition = m_BaseBytePosition; + newInfo.version = 1; + #if UNITY_EDITOR + newInfo.lookupCount = 0; + #endif + newInfo.currentTypeName = typeString; + #if !UNITY_RELEASE + newInfo.currentTypeNameCheck = typeString; + #endif + newInfo.cachedIterator = newInfo.type->begin(); + newInfo.cachedBytePosition = m_BaseBytePosition; + m_StackInfo.push(newInfo); + m_CurrentStackInfo = &m_StackInfo.top(); + + return kMatchesType; + } + + TypeTree::TypeTreeList::const_iterator c; + + StackedInfo& info = *m_CurrentStackInfo; + + const TypeTree::TypeTreeList& children = info.type->m_Children; + + // Start searching at the cached position + SInt32 newBytePosition = info.cachedBytePosition; + int count = 0; + for (c=info.cachedIterator;c != children.end ();++c) + { + if (c->m_Name == name) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + count++; + } + + #if UNITY_EDITOR + if (count > 1) + m_TypeTreeHasChanged = true; + #endif + + // Didn't find it, try again starting at the first child + if (c == children.end ()) + { + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + + // Find name conversion lookup for this type + #if !UNITY_RELEASE + DebugAssertIf(info.currentTypeName != info.currentTypeNameCheck); + #endif + const AllowNameConversion::mapped_type* nameConversion = GetAllowedNameConversions(info.currentTypeName, name); + + newBytePosition = info.bytePosition; + for (c=children.begin();c != children.end();++c) + { + if (c->m_Name == name) + break; + if (nameConversion && nameConversion->count(const_cast<char*>(c->m_Name.c_str()))) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + } + + // No child with name was found? + if (c == children.end ()) + { + #if LOG_MISSING_VARIBALES + string s ("Variable not found in old file "); + GetTypePath (m_StackInfo.top ().type, s); + s = s + " new name and type: " + name; + s = s + '(' + typeString + ")\n"; + m_OldBaseType->DebugPrint (s); + LogString (s); + #endif + + return kNotFound; + } + } + + #if UNITY_EDITOR + m_CurrentStackInfo->lookupCount++; + #endif + + info.cachedIterator = c; + info.cachedBytePosition = newBytePosition; + + /*Unoptimized version: + + // Find name in children typeTree, updating position + SInt32 newBytePosition = info.bytePosition; + + // Find name conversion lookup for this type + const AllowNameConversion::mapped_type* nameConversion = NULL; + DebugAssertIf(info.currentTypeName != info.currentTypeNameCheck); + AllowNameConversion::iterator foundNameConversion = gAllowNameConversion.find(make_pair(const_cast<char*>(info.currentTypeName), const_cast<char*>(name))); + if (foundNameConversion == gAllowNameConversion.end()) + nameConversion = &foundNameConversion->second; + + for (c=children.begin ();c != children.end ();++c) + { + if (c->m_Name == name) + break; + if (nameConversion && nameConversion->count(const_cast<char*>(c->m_Name.c_str()))) + break; + + // Walk through old typetree, updating position + Walk (*c, &newBytePosition); + } + + // No child with name was found? + if (c == children.end ()) + { + #if LOG_MISSING_VARIBALES + string s ("Variable not found in old file "); + GetTypePath (m_OldType.top (), s); + s = s + " new name and type: " + name; + s = s + '(' + typeString + ")\n"; + m_OldBaseType->DebugPrint (s); + AssertStringQuiet (s); + #endif + + return kNotFound; + } + */ + + // Walk trough the already iterated array elements + if (info.type->m_IsArray && c != children.begin ()) + { + SInt32 arrayPosition = *m_CurrentPositionInArray; + + // There are no arrays in the subtree so + // we can simply use the cached bytesize + // Alignment cuts across this so use the slow path in that case + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + { + newBytePosition += c->m_ByteSize * arrayPosition; + } + // Walk through old typetree, updating position + else + { + ArrayPositionInfo& arrayInfo = m_PositionInArray.top(); + SInt32 cachedArrayPosition = 0; + if (arrayInfo.cachedArrayPosition <= arrayPosition) + { + newBytePosition = arrayInfo.cachedBytePosition; + cachedArrayPosition = arrayInfo.cachedArrayPosition; + } + + for (SInt32 i = cachedArrayPosition;i < arrayPosition;i++) + Walk (*c, &newBytePosition); + + arrayInfo.cachedArrayPosition = arrayPosition; + arrayInfo.cachedBytePosition = newBytePosition; + } + + (*m_CurrentPositionInArray)++; + } + + StackedInfo newInfo; + newInfo.type = &*c; + newInfo.bytePosition = newBytePosition; + newInfo.version = 1; + #if UNITY_EDITOR + newInfo.lookupCount = 0; + #endif + newInfo.cachedIterator = newInfo.type->begin(); + newInfo.cachedBytePosition = newBytePosition; + newInfo.currentTypeName = typeString; + #if !UNITY_RELEASE + newInfo.currentTypeNameCheck = typeString; + #endif + + m_StackInfo.push(newInfo); + m_CurrentStackInfo = &m_StackInfo.top(); + + int conversion = kNeedConversion; + // Does the type match (compare type string) + // The root type should get a transfer in any case because the type might change + // Eg. TransformComponent renamed to Transform (Typename mismatch but we still want to serialize) + if (c->m_Type == typeString || m_StackInfo.size () == 1) + { + conversion = kMatchesType; + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + conversion = kFastPathMatchesType; + } + else if (AllowTypeNameConversion (c->m_Type, typeString)) + { + conversion = kMatchesType; + if (c->m_ByteSize != -1 && (c->m_MetaFlag & (kAnyChildUsesAlignBytesFlag | kAlignBytesFlag)) == 0) + conversion = kFastPathMatchesType; + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + } + else + { + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + } + + if (conversion == kNeedConversion && converter != NULL) + *converter = FindConverter(c->m_Type.c_str(), typeString); + + return conversion; +} + +void SafeBinaryRead::SetVersion (int version) +{ + m_CurrentStackInfo->version = version; +} + + +void SafeBinaryRead::EndTransfer () +{ + #if UNITY_EDITOR + if (m_CurrentStackInfo && m_CurrentStackInfo->lookupCount != m_CurrentStackInfo->type->m_Children.size()) + { + m_TypeTreeHasChanged = true; + } + #endif + + m_StackInfo.pop(); + if (!m_StackInfo.empty()) + { + m_CurrentStackInfo = &m_StackInfo.top(); + } + else + m_CurrentStackInfo = NULL; + + m_DidReadLastProperty = true; +} + +bool SafeBinaryRead::BeginArrayTransfer (const char* name, const char* typeString, SInt32& size) +{ + if (BeginTransfer (name, typeString, NULL) == kNotFound) + return false; + + Transfer (size, "size"); + ArrayPositionInfo info; + info.arrayPosition = 0; + info.cachedBytePosition = -1; + info.cachedArrayPosition = std::numeric_limits<SInt32>::max(); + m_PositionInArray.push (info); + m_CurrentPositionInArray = &m_PositionInArray.top().arrayPosition; + + Assert (GetActiveOldTypeTree ().m_Children.front ().m_Name == "size"); + + return true; +} + +void SafeBinaryRead::EndArrayTransfer () +{ + m_PositionInArray.pop (); + if (!m_PositionInArray.empty()) + m_CurrentPositionInArray = &m_PositionInArray.top().arrayPosition; + else + m_CurrentPositionInArray = NULL; + + #if UNITY_EDITOR + m_TypeTreeHasChanged = true; + #endif + + EndTransfer (); +} + +bool SafeBinaryRead::IsCurrentVersion () +{ + return m_CurrentStackInfo->version == m_CurrentStackInfo->type->m_Version; +} + +bool SafeBinaryRead::IsOldVersion (int version) +{ + return m_CurrentStackInfo->type->m_Version == version; +} + +bool SafeBinaryRead::IsVersionSmallerOrEqual (int version) +{ + return m_CurrentStackInfo->type->m_Version <= version; +} + +void SafeBinaryRead::TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaflag) +{ + SInt32 size; + if (!BeginArrayTransfer (name, "TypelessData", size)) + { + *byteSize = 0; + return; + } + // We can only transfer the array if the size was transferred as well + AssertIf (GetActiveOldTypeTree ().m_Children.front ().m_Name != "size"); + + *byteSize = size; + + EndArrayTransfer (); +} + +void SafeBinaryRead::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + if (copyData == NULL || byteSize == 0) return; + + m_Cache.Read (copyData, byteSize); +} + +#endif // SUPPORT_SERIALIZED_TYPETREES diff --git a/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h new file mode 100644 index 0000000..e6cd5a3 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SafeBinaryRead.h @@ -0,0 +1,290 @@ +#ifndef SAFEBINARYREAD_H +#define SAFEBINARYREAD_H + +#include "Configuration/UnityConfigure.h" + +#if SUPPORT_SERIALIZED_TYPETREES + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + +#include <stack> +class dynamic_bitset; +class SafeBinaryRead; + +#define LOG_CONVERTING_VARIBALES 0 +#define LOG_MISSING_VARIBALES 0 + +typedef bool ConversionFunction (void* inData, SafeBinaryRead& transfer); + +class EXPORT_COREMODULE SafeBinaryRead : public TransferBase +{ + CachedReader m_Cache; + SInt32 m_BaseBytePosition; + SInt32 m_BaseByteSize; + + const TypeTree* m_OldBaseType; + #if UNITY_EDITOR + bool m_TypeTreeHasChanged; + #endif + + enum { kNotFound = 0, kMatchesType = 1, kFastPathMatchesType = 2, kNeedConversion = -1 }; + + struct StackedInfo + { + const TypeTree* type; /// The type tree of the old type we are reading data from + const char* currentTypeName; /// The name of the type we are currently reading (This is the new type name and not from the stored data) + int bytePosition;/// byte position of that element + int version; /// current version (This is the new version and not from the stored data) + + int cachedBytePosition; /// The cached byte position of the last visited child + TypeTree::const_iterator cachedIterator; /// The cached iterator of the last visited child + #if UNITY_EDITOR + int lookupCount; // counts number of looks, used to determine if the typetree matches + #endif + + #if !UNITY_RELEASE + std::string currentTypeNameCheck;/// For debugging purposes in case someone changes the typename string while still reading! + #endif + + }; + + StackedInfo* m_CurrentStackInfo; + SInt32* m_CurrentPositionInArray; + std::stack<StackedInfo> m_StackInfo; + + struct ArrayPositionInfo + { + SInt32 arrayPosition; + SInt32 cachedBytePosition; + SInt32 cachedArrayPosition; + }; + + std::stack<ArrayPositionInfo> m_PositionInArray;// position in an array + + bool m_DidReadLastProperty; + + friend class MonoBehaviour; + +public: + + CachedReader& Init (const TypeTree& oldBase, int bytePosition, int byteSize, int flags); + CachedReader& Init (SafeBinaryRead& transfer); + ~SafeBinaryRead (); + + void SetVersion (int version); + bool IsCurrentVersion (); + bool IsOldVersion (int version); + bool IsVersionSmallerOrEqual (int version); + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool IsReadingBackwardsCompatible () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return m_Flags & kSwapEndianess; } + + bool DidReadLastProperty () { return m_DidReadLastProperty; } + bool DidReadLastPPtrProperty () { return m_DidReadLastProperty; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + /// In order to transfer typeless data (Read: transfer data real fast) + /// Call TransferTypeless. You have to always do this. Even for a proxytransfer. Then when you want to access the datablock. + /// Call TransferTypelessData + /// On return: + /// When reading bytesize will contain the size of the data block that should be read, + /// when writing bytesize has to contain the size of the datablock. + /// MarkerID will contain an marker which you have to give TransferTypelessData when you want to start the actual transfer. + /// optional: A serializedFile will be seperated into two chunks. One is the normal object data. (It is assumed that they are all relatively small) + /// So caching them makes a lot of sense. Big datachunks will be stored in another part of the file. + /// They will not be cached but usually read directly into the allocated memory, probably reading them asynchronously + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + // markerID is the id that was given by TransferTypeless. + // byteStart is the bytestart relative to the beginning of the typeless data + // copyData is a pointer to where the data will be written or read from + /// optional: if metaFlag is kTransferBigData the data will be optimized into seperate blocks, + void TransferTypelessData (unsigned byteSize, void* copyData, int metaData = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + bool GetTransferFileInfo(unsigned* position, const char** filePath) const; + + const TypeTree& GetActiveOldTypeTree () { return *m_CurrentStackInfo->type; } + + static void RegisterConverter (const char* oldType, const char* newType, ConversionFunction* converter); + static void CleanupConverterTable (); + + #if UNITY_EDITOR + /// Returns if the typetree is different from what was loaded in. + /// Currently this is incomplete. Arrays will always return true. + bool HasDifferentTypeTree () + { + return m_TypeTreeHasChanged; + } + #endif + +private: + + // BeginTransfer / EndTransfer + int BeginTransfer (const char* name, const char* typeString, ConversionFunction** converter); + bool BeginArrayTransfer (const char* name, const char* typeString, SInt32& size); + + // Override the root type name, this is used by scripts that can only determine the class type name after the mono class has actually been loaded + void OverrideRootTypeName (const char* typeString); + + void EndTransfer (); + void EndArrayTransfer (); + + void Walk (const TypeTree& typeTree, SInt32* bytePosition); +}; + +template<class T>inline +void SafeBinaryRead::TransferBasicData (T& data) +{ + m_Cache.Read (data, m_CurrentStackInfo->bytePosition); + if (ConvertEndianess()) + { + SwapEndianBytes (data); + } +} + +template<class T> inline +void SafeBinaryRead::TransferSTLStyleArray (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + if (!BeginArrayTransfer ("Array", "Array", size)) + return; + + SerializeTraits<T>::ResizeSTLStyleArray (data, size); + + typename T::iterator i; + typename T::iterator end = data.end (); + if (size != 0) + { + int conversion = BeginTransfer ("data", SerializeTraits<typename T::value_type>::GetTypeString(&*data.begin()), NULL); + int elementSize = m_CurrentStackInfo->type->m_ByteSize; + *m_CurrentPositionInArray = 0; + // If the data types are matching and element size can be determined + // then we fast path the whole thing and skip all the duplicate stack walking + if (conversion == kFastPathMatchesType) + { + int basePosition = m_CurrentStackInfo->bytePosition; + + for (i = data.begin ();i != end;++i) + { + int currentBytePosition = basePosition + (*m_CurrentPositionInArray) * elementSize; + m_CurrentStackInfo->cachedBytePosition = currentBytePosition; + m_CurrentStackInfo->bytePosition = currentBytePosition; + m_CurrentStackInfo->cachedIterator = m_CurrentStackInfo->type->begin(); + (*m_CurrentPositionInArray)++; + SerializeTraits<typename T::value_type>::Transfer (*i, *this); + } + EndTransfer(); + } + // Fall back to converting variables + else + { + EndTransfer(); + for (i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } + } + + EndArrayTransfer (); +} + +template<class T> inline +void SafeBinaryRead::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + if (!BeginArrayTransfer ("Array", "Array", size)) + return; + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typename NonConstContainerValueType<T>::value_type p; + + data.clear (); + for (int i=0;i<size;i++) + { + Transfer (p, "data"); + data.insert (p); + } + EndArrayTransfer (); +} + +template<class T> inline +void SafeBinaryRead::TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags) +{ + ConversionFunction* converter; + int conversion = BeginTransfer (name, typeName, &converter); + if (conversion == kNotFound) + return; + + if (conversion >= kMatchesType) + SerializeTraits<T>::Transfer (data, *this); + // Try conversion + else + { + bool success = false; + if (converter != NULL) + success = converter (&data, *this); + + #if LOG_CONVERTING_VARIBALES + { + string s ("Converting variable "); + if (success) + s += " succeeded "; + else + s += " failed "; + + GetTypePath (m_OldType.top (), s); + s = s + " new type: "; + s = s + " new type: (" + SerializeTraits<T>::GetTypeString () + ")\n"; + m_OldBaseType->DebugPrint (s); + AssertStringQuiet (s); + } + #endif + } + EndTransfer (); +} + +template<class T> inline +void SafeBinaryRead::Transfer (T& data, const char* name, TransferMetaFlags) +{ + TransferWithTypeString(data, name, SerializeTraits<T>::GetTypeString(&data), kNoTransferFlags); +} + +#else + +namespace SafeBinaryReadManager +{ + inline void StaticInitialize(){}; + inline void StaticDestroy(){}; +} + +class SafeBinaryRead +{ +public: + static void RegisterAllowTypeNameConversion (const char* oldTypeName, const char* newTypeName) { } + static void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName) { } + static void CleanupConverterTable() { } +}; + +#endif +#endif diff --git a/Runtime/Serialize/TransferFunctions/SerializeTransfer.h b/Runtime/Serialize/TransferFunctions/SerializeTransfer.h new file mode 100644 index 0000000..f96d7d5 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/SerializeTransfer.h @@ -0,0 +1,14 @@ +#ifndef SERIALIZETRANSFER_H +#define SERIALIZETRANSFER_H + +#if SUPPORT_TEXT_SERIALIZATION +#include "YAMLRead.h" +#include "YAMLWrite.h" +#endif +#include "RemapPPtrTransfer.h" +#include "StreamedBinaryRead.h" +#include "ProxyTransfer.h" +#include "SafeBinaryRead.h" +#include "StreamedBinaryWrite.h" + +#endif diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp new file mode 100644 index 0000000..4daa8fb --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.cpp @@ -0,0 +1,54 @@ +#include "UnityPrefix.h" +#include "StreamedBinaryRead.h" + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::Align () +{ + m_Cache.Align4Read(); +} + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/) +{ + SInt32 size; + Transfer (size, "size"); + *byteSize = size; +} + +// markerID is the id that was given by TransferTypeless. +// optional copyData: is a pointer to where the data will be written or read from +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData) +{ + if (byteSize == 0) + return; + + if (copyData == NULL) + { + // seek byte + m_Cache.Skip(byteSize); + } + else + m_Cache.Read (copyData, byteSize); + Align(); +} + +template <bool kSwapEndianess> +void StreamedBinaryRead<kSwapEndianess>::ReadDirect (void* data, int byteSize) +{ + AssertIf (kSwapEndianess); + m_Cache.Read (data, byteSize); +} + + +template void StreamedBinaryRead<true>::ReadDirect (void* data, int byteSize); +template void StreamedBinaryRead<false>::ReadDirect (void* data, int byteSize); + +template void StreamedBinaryRead<true>::Align(); +template void StreamedBinaryRead<false>::Align(); + +template void StreamedBinaryRead<true>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); +template void StreamedBinaryRead<false>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); + +template void StreamedBinaryRead<true>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); +template void StreamedBinaryRead<false>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h new file mode 100644 index 0000000..81a015c --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryRead.h @@ -0,0 +1,174 @@ +#ifndef STREAMEDBINARYREAD_H +#define STREAMEDBINARYREAD_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + + +template<bool kSwapEndianess> +class StreamedBinaryRead : public TransferBase +{ + CachedReader m_Cache; + + friend class MonoBehaviour; + +public: + + CachedReader& Init (int flags) { m_UserData = NULL; m_Flags = flags; return m_Cache; } + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return kSwapEndianess; } + + bool DidReadLastProperty () { return true; } + bool DidReadLastPPtrProperty () { return true; } + + void EnableResourceImage (ActiveResourceImage targetResourceImage) { m_Cache.BeginResourceImage(targetResourceImage); } + bool ReadStreamingInfo(StreamingInfo* streamingInfo); + + bool ShouldChannelOverride (); + CachedReader& GetCachedReader () { return m_Cache; } + + const char* GetSerializedFilePathName() { return m_Cache.GetSerializedFilePathName(); } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + // markerID is the id that was given by TransferTypeless. + // optional copyData: is a pointer to where the data will be written or read from + void TransferTypelessData (unsigned byteSize, void* copyData, int metaData = 0); + + /// Reads byteSize bytes into data. This may onle be used if UseOptimizedReading returns true. + void EXPORT_COREMODULE ReadDirect (void* data, int byteSize); + + void EXPORT_COREMODULE Align (); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); +}; + +template <bool kSwapEndianess> +bool StreamedBinaryRead<kSwapEndianess>::ReadStreamingInfo(StreamingInfo* streamingInfo) +{ + Assert(streamingInfo != NULL); + + if (!m_Cache.IsReadingResourceImage()) + return false; + + // Read the size & offset values from the serialized file + // The size & offset describes where the data is in the streamed file + UInt32 offset, size; + Transfer (size, "ri_size"); + Transfer (offset, "ri_offset"); + + m_Cache.GetStreamingInfo (offset, size, streamingInfo); + return true; +} + + + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferSTLStyleArray (T& data, TransferMetaFlags /*metaFlags*/) +{ + if (m_Cache.IsReadingResourceImage()) + { + // Read the size & offset from the serialized file + UInt32 offset, size; + Transfer (size, "ri_size"); + Transfer (offset, "ri_offset"); + + // Fetch the pointer from the pre-loaded resource image. + unsigned bufferSize = sizeof (typename T::value_type) * size; + UInt8* buffer = m_Cache.FetchResourceImageData (offset, bufferSize); + SerializeTraits<T>::resource_image_assign_external (data, buffer, buffer + bufferSize); + + m_Cache.EndResourceImage(); + } + else + { + SInt32 size; + Transfer (size, "size"); + + SerializeTraits<T>::ResizeSTLStyleArray (data, size); + + if (!kSwapEndianess && SerializeTraits<typename T::value_type>::AllowTransferOptimization () && SerializeTraits<T>::IsContinousMemoryArray ()) + { + //AssertIf (size != distance (data.begin (), data.end ())); + if( size != 0 ) + ReadDirect (&*data.begin (), size * sizeof (typename T::value_type)); + } + else + { + typename T::iterator i; + typename T::iterator end = data.end (); + //AssertIf (size != distance (data.begin (), end)); + for (i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } + } +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size; + Transfer (size, "size"); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typename NonConstContainerValueType<T>::value_type p; + + data.clear (); + for (int i=0;i<size;i++) + { + Transfer (p, "data"); + data.insert (p); + } +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::Transfer (T& data, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> +void StreamedBinaryRead<kSwapEndianess>::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryRead<kSwapEndianess>::TransferBasicData (T& data) +{ + AssertIf (sizeof (T) > 8); + m_Cache.Read (data); + if (kSwapEndianess) + { + SwapEndianBytes (data); + } +} +#endif + + + diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp new file mode 100644 index 0000000..a7699e9 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.cpp @@ -0,0 +1,76 @@ +#include "UnityPrefix.h" +#include "StreamedBinaryWrite.h" +#include "Configuration/UnityConfigure.h" + +template <bool kSwapEndianess> +CachedWriter& StreamedBinaryWrite<kSwapEndianess>::Init (int flags, BuildTargetSelection target) +{ + m_Flags = flags; + m_UserData = NULL; + m_Target = target; + +#if UNITY_EDITOR && CHECK_SERIALIZE_ALIGNMENT + m_Cache.SetCheckSerializeAlignment(true); + #endif + return m_Cache; +} + +template <bool kSwapEndianess> +CachedWriter& StreamedBinaryWrite<kSwapEndianess>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag) +{ + m_Flags = flags; + m_Target = target; + m_Cache = cachedWriter; + m_UserData = NULL; + + #if UNITY_EDITOR + m_BuildUsageTag = buildUsageTag; + #endif + +#if UNITY_EDITOR && CHECK_SERIALIZE_ALIGNMENT + m_Cache.SetCheckSerializeAlignment(true); +#endif + return m_Cache; +} + +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::Align () +{ + m_Cache.Align4Write(); +} + + +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::TransferTypeless (unsigned* byteSize, const char* /* name*/, TransferMetaFlags/* metaFlag*/) +{ + SInt32 size = *byteSize; + Transfer (size, "size"); +} + +// markerID is the id that was given by TransferTypeless. +// byteStart is +// optional temporaryDataHandle: temporaryDataHandle is a handle to the data +// optional copyData: is a pointer to where the data will be written or read from +template <bool kSwapEndianess> +void StreamedBinaryWrite<kSwapEndianess>::TransferTypelessData (unsigned byteSize, void* copyData, int/* metaData*/) +{ + AssertIf(copyData == NULL && byteSize != 0); + m_Cache.Write (copyData, byteSize); + Align(); +} + + +template CachedWriter& StreamedBinaryWrite<false>::Init (int flags, BuildTargetSelection target); +template CachedWriter& StreamedBinaryWrite<true>::Init (int flags, BuildTargetSelection target); + +template CachedWriter& StreamedBinaryWrite<false>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); +template CachedWriter& StreamedBinaryWrite<true>::Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); + +template void StreamedBinaryWrite<false>::Align (); +template void StreamedBinaryWrite<true>::Align (); + +template void StreamedBinaryWrite<false>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); +template void StreamedBinaryWrite<true>::TransferTypeless (unsigned* byteSize, const char*/* name*/, TransferMetaFlags/* metaFlag*/); + +template void StreamedBinaryWrite<false>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); +template void StreamedBinaryWrite<true>::TransferTypelessData (unsigned byteSize, void* copyData, int metaData); diff --git a/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h new file mode 100644 index 0000000..b5d9074 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/StreamedBinaryWrite.h @@ -0,0 +1,187 @@ +#ifndef STREAMEDBINARYWRITE_H +#define STREAMEDBINARYWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "Runtime/Serialize/CacheWrap.h" +#include "Runtime/Serialize/SwapEndianBytes.h" + +template<bool kSwapEndianess> +class EXPORT_COREMODULE StreamedBinaryWrite : public TransferBase +{ + CachedWriter m_Cache; + BuildTargetSelection m_Target; + + #if UNITY_EDITOR + BuildUsageTag m_BuildUsageTag; + #endif + friend class MonoBehaviour; + +public: + + CachedWriter& Init (int flags, BuildTargetSelection target); + CachedWriter& Init (const CachedWriter& cachedWriter, int flags, BuildTargetSelection target, const BuildUsageTag& buildUsageTag); + + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool ConvertEndianess () { return kSwapEndianess; } + bool IsWritingGameReleaseData () + { + return IsSerializingForGameRelease (); + } + bool IsBuildingTargetPlatform (BuildTargetPlatform platform) + { + #if UNITY_EDITOR + if (platform == kBuildAnyPlayerData) + return m_Target.platform >= kBuildValidPlayer; + else + return m_Target.platform == platform; + #else + return false; + #endif + } + + #if UNITY_EDITOR + BuildUsageTag GetBuildUsage () + { + return m_BuildUsageTag; + } + #endif + + BuildTargetSelection GetBuildingTarget () { return m_Target; } + + template<class T> + void Transfer (T& data, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferWithTypeString (T& data, const char* name, const char* typeName, TransferMetaFlags metaFlag = kNoTransferFlags); + + void EnableResourceImage (ActiveResourceImage targetResourceImage) + { + #if UNITY_EDITOR + m_Cache.BeginResourceImage (targetResourceImage); + #endif + } + + /// In order to transfer typeless data (Read: transfer data real fast) + /// Call TransferTypeless. You have to always do this. Even for a proxytransfer. Then when you want to access the datablock. + /// Call TransferTypelessData + /// On return: + /// When reading bytesize will contain the size of the data block that should be read, + /// when writing bytesize has to contain the size of the datablock. + /// MarkerID will contain an marker which you have to give TransferTypelessData when you want to start the actual transfer. + /// optional: A serializedFile will be seperated into two chunks. One is the normal object data. (It is assumed that they are all relatively small) + /// So caching them makes a lot of sense. Big datachunks will be stored in another part of the file. + /// They will not be cached but usually read directly into the allocated memory, probably reading them asynchronously + void TransferTypeless (unsigned* byteSize, const char* name, TransferMetaFlags metaFlag = kNoTransferFlags); + + // markerID is the id that was given by TransferTypeless. + // copyData is a pointer to where the data will be written or read from + void TransferTypelessData (unsigned byteSize, void* copyData, int metaFlag = 0); + + bool GetTransferFileInfo(unsigned* position, const char** filePath) const; + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferSTLStyleArray (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + template<class T> + void TransferSTLStyleMap (T& data, TransferMetaFlags metaFlag = kNoTransferFlags); + + void Align (); + + CachedWriter& GetCachedWriter() { return m_Cache; } +}; + + + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferSTLStyleArray (T& data, TransferMetaFlags /*metaFlags*/) +{ + #if UNITY_EDITOR + if (m_Cache.IsWritingResourceImage()) + { + // Grab the offset where the resourceImage is currently at + UInt32 offsetInResourceImage = m_Cache.GetPosition(); + + // Write the actual data to the resource image + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + Transfer (*i, "data"); + + Assert (m_Cache.IsWritingResourceImage()); + m_Cache.EndResourceImage (); + Assert (!m_Cache.IsWritingResourceImage()); + + UInt32 size = data.size (); + + // Writ ethe size & offset to the serialized file + Transfer (size, "ri_size"); + Transfer (offsetInResourceImage, "ri_offset"); + } + else + #endif + { + SInt32 size = data.size (); + Transfer (size, "size"); + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + Transfer (*i, "data"); + } +} + + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferSTLStyleMap (T& data, TransferMetaFlags) +{ + SInt32 size = data.size (); + Transfer (size, "size"); + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + + typename T::iterator end = data.end (); + for (typename T::iterator i = data.begin ();i != end;++i) + { + non_const_value_type& p = (non_const_value_type&)(*i); + Transfer (p, "data"); + } +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::Transfer (T& data, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferWithTypeString (T& data, const char*, const char*, TransferMetaFlags) +{ + SerializeTraits<T>::Transfer (data, *this); +} + +template<bool kSwapEndianess> +template<class T> inline +void StreamedBinaryWrite<kSwapEndianess>::TransferBasicData (T& data) +{ + if (kSwapEndianess) + { + T temp = data; + SwapEndianBytes (temp); + m_Cache.Write (temp); + } + else + m_Cache.Write (data); +} + +#endif diff --git a/Runtime/Serialize/TransferFunctions/TransferBase.h b/Runtime/Serialize/TransferFunctions/TransferBase.h new file mode 100644 index 0000000..aae527a --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferBase.h @@ -0,0 +1,177 @@ +#ifndef TRANSFERBASE_H +#define TRANSFERBASE_H + +#include "Runtime/Serialize/SerializeTraits.h" +#include "Runtime/Serialize/SerializationMetaFlags.h" + + +struct ReduceCopyData; +struct StreamingInfo; + + +class EXPORT_COREMODULE TransferBase +{ +public: + + TransferBase () + : m_Flags (0) + , m_UserData (NULL) {} + + /// @name Predicates + /// @{ + + /// Get the TransferInstructionFlags. + /// Commonly used to special case transfer functions for specific operations. + int GetFlags () { return m_Flags; } + + /// If true, the transfer is reading data from a source. (Could be fread from a file or reading from a block of memory) + /// @note There are transfers for which neither IsReading() nor IsWriting() is true (for example when generating a typetree). + /// IsReading is NOT the inverse of IsWriting. + bool IsReading () { return false; } + + /// If true, the transfer reads PPtrs (Object references) + /// This is true when reading from a memory stream or file but also when using RemapPPtrTransfer (A generic way of iterating all object references) + bool IsReadingPPtr () { return false; } + + /// Whether the last Transfer() resulted in a value store, i.e. had actual data + /// transfered from the stream. + /// It is important to use this function instead of IsReading when + /// When reading from a stream that does not define all the data, the desired behaviour is that default values from constructor are fully preserved. + /// All transfer functions do this internally (transferred properties are left untouched when the data does not exist for example in a Yaml file) + /// But when the serialized property needs to be manually converted in the Transfer function, then it is important to check if the value was actually read. + /// CODE EXAMPLE: + /// bool enabled; + /// if (transfer.IsWriting ()) + /// enabled = m_Flags == 1; + /// TRANSFER (enabled); + /// if (transfer.DidReadLastProperty ()) + /// m_Flags = enabled ? 1 : 0; + bool DidReadLastProperty () const { return false; } + + /// Same as DidReadLastProperty, but only returns true when reading PPtr properties. + /// A compile time optimization necessary for removing generated code by RemapPPtrTransfer. + bool DidReadLastPPtrProperty () const { return false; } + + + /// If true, the transfer is writing out data. + bool IsWriting () { return false; } + /// If true, the transfer is writing out PPtr data. This is true when writing to a memory stream or file, but also when using RemapPPtrTransfer. + bool IsWritingPPtr () { return false; } + + /// Are we reading data from a data source that is not guaranteed to have the same data layout as the Transfer function. + /// eg. StreamedBinaryRead always returns false. YamlRead & SafeBinaryRead return true. + bool IsReadingBackwardsCompatible () { return false; } + + /// When writing or reading from disk we need to translate instanceID + /// to LocalIdentifierInFile & LocalSerializedFileIndex or in the case of Yaml files, guids + LocalIdentifierInFile. + /// This returns true when remapping of the instanceID should be performed. + bool NeedsInstanceIDRemapping () { return false; } + + /// Are we transferring data with endianess swapping. (We might neither endianess swap on write or read based on IsReading / IsWriting) + /// The endianess conversion is done by the TransferBackend, but there are some special cases where you might want to handle it yourself. + /// (For example a texture data is transferred a single UInt8* array, so all endianess swapping is the responsibiltiy of the texture transfer function.) + bool ConvertEndianess () { return false; } + + /// Are we reading/writing a .meta file (Asset importers use it to differentiate reading/writing of a Library/metadata cached file. ) + /// @TODO: We should rename Library/metadata to Library/cachedata and cleanup the usage of metadata vs .meta file. It is confusing. + bool AssetMetaDataOnly () { return false; } + + /// Is this a RemapPPtrTransfer backend. Commonly used to do very specialized code when generating dependencies using RemapPPtrTransfer. + bool IsRemapPPtrTransfer () { return false; } + + /// Return true if we are writing the data for a player. + bool IsWritingGameReleaseData () { return false; } + + /// Are we serializing data for use by the player. + /// This includes reading/writing/generating typetrees. And can be when creating data from the editor for player or when simply reading/writing data in the player. + /// Commonly used to not serialize data that does not exist in the player. + bool IsSerializingForGameRelease () + { + #if UNITY_EDITOR + return m_Flags & kSerializeGameRelease; + #else + return true; + #endif + } + + /// @} + + /// @name Build Targets + /// @{ + + /// Returns true in the editor when writing the data for a player of the specified target platform. + bool IsBuildingTargetPlatform (BuildTargetPlatform) { return false; } + /// Returns the target platform we are building for. Only returns the target platform when writing data. + BuildTargetSelection GetBuildingTarget () { return BuildTargetSelection::NoTarget (); } + + #if UNITY_EDITOR + /// BuildUsageTag carries information generated by the build process about the object being serialized. + /// For example the buildpipeline might instruct the transfer system to strip normals and tangents from a Mesh, + /// because it knows that no renderers & materials in all scenes actually use them. + BuildUsageTag GetBuildUsage () { return BuildUsageTag (); } + #endif + + /// @} + + /// @name Versioning + /// @{ + + /// Sets the "version of the class currently transferred" + void SetVersion (int) {} + + /// Returns if the transferred data's version is the version used by the source code + bool IsVersionSmallerOrEqual (int /*version*/) { return false; } + + /// Deprecated: use IsVersionSmallerOrEqual instead. + bool IsOldVersion (int /*version*/) { return false; } + bool IsCurrentVersion () { return true; } + + /// @} + + /// @name Transfers + /// @{ + + /// Alignment in the serialization system is done manually. + /// The serialization system only ever cares about 4 byte alignment. + /// When writing data that has an alignment of less than 4 bytes, followed by data that has 4 byte alignment, + /// then Align must be called before the 4 byte aligned data. + /// TRANSFER (1byte); + /// TRANSFER (1byte); + /// transfer.Align (); + /// TRANSFER (4byte); + void Align () {} + + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void TransferBasicData (T&) { } + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void TransferPtr (bool, ReduceCopyData*) {} + + /// Internal function. Should only be called from SerializeTraits + template<class T> + void ReduceCopy (const ReduceCopyData&){} + /// @} + + /// user data. + void* GetUserData () { return m_UserData; } + void SetUserData (void* userData) { m_UserData = userData; } + + void AddMetaFlag(int /*mask*/) {} + + /// Deprecated + void BeginMetaGroup (std::string /*name*/) {} + void EndMetaGroup () {} + void EnableResourceImage (ActiveResourceImage /*targetResourceImage*/) {} + bool ReadStreamingInfo (StreamingInfo* /*streamingInfo*/) { return false; } + bool NeedNonCriticalMetaFlags () { return false; } + +protected: + + int m_Flags; + void* m_UserData; +}; + +#endif // !TRANSFER_BASE diff --git a/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp b/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp new file mode 100644 index 0000000..35ae03f --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferNameConversions.cpp @@ -0,0 +1,76 @@ +#include "UnityPrefix.h" +#include "TransferNameConversions.h" +#include "Runtime/Utilities/InitializeAndCleanup.h" + + +TranferNameConversionsManager::TranferNameConversionsManager() +{ + m_AllowTypeNameConversions = UNITY_NEW(AllowTypeNameConversions,kMemSerialization); + m_AllowNameConversion = UNITY_NEW(AllowNameConversion,kMemSerialization); +} + +TranferNameConversionsManager::~TranferNameConversionsManager() +{ + UNITY_DELETE(m_AllowTypeNameConversions,kMemSerialization); + UNITY_DELETE(m_AllowNameConversion,kMemSerialization); +} + +TranferNameConversionsManager* TranferNameConversionsManager::s_Instance = NULL; +void TranferNameConversionsManager::StaticInitialize() +{ + s_Instance = UNITY_NEW_AS_ROOT(TranferNameConversionsManager, kMemManager, "SerializationBackwardsCompatibility", ""); +} +void TranferNameConversionsManager::StaticDestroy() +{ + UNITY_DELETE(s_Instance, kMemManager); +} +static RegisterRuntimeInitializeAndCleanup s_TranferNameConversionsManagerCallbacks(TranferNameConversionsManager::StaticInitialize, TranferNameConversionsManager::StaticDestroy); + +bool AllowTypeNameConversion (const UnityStr& oldType, const char* newTypeName) +{ + pair<AllowTypeNameConversions::iterator, AllowTypeNameConversions::iterator> range; + range = GetTranferNameConversionsManager().m_AllowTypeNameConversions->equal_range (const_cast<char*>(oldType.c_str())); + for (;range.first != range.second;range.first++) + { + if (strcmp(range.first->second, newTypeName) == 0) + return true; + } + + // Special support for Mono PPtr's + // With Unity 1.6 MonoBehaviour pointers have a special prefix and keep the class name in the PPtr. [ PPtr<$MyClass> ] + // With Unity 1.5.1 it was simply PPtr<MonoBehaviour>. This made correct typechecking unneccessarily hard. + // Here we provide backwards compatibility with the old method. + if (strncmp("PPtr<$", newTypeName, 6) == 0) + { + if (oldType.find("PPtr<") == 0) + return true; + } + + return false; +} + +const AllowNameConversion::mapped_type* GetAllowedNameConversions (const char* type, const char* name) +{ + const AllowNameConversion::mapped_type* nameConversion = NULL; + AllowNameConversion::iterator foundNameConversion = GetTranferNameConversionsManager().m_AllowNameConversion->find(make_pair(const_cast<char*>(type), const_cast<char*>(name))); + if (foundNameConversion != GetTranferNameConversionsManager().m_AllowNameConversion->end()) + nameConversion = &foundNameConversion->second; + return nameConversion; +} + +void RegisterAllowTypeNameConversion (const char* from, const char* to) +{ + GetTranferNameConversionsManager().m_AllowTypeNameConversions->insert(make_pair(const_cast<char*>(from), const_cast<char*>(to))); +} + +void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName) +{ + AllowNameConversion::mapped_type& allowed = (*GetTranferNameConversionsManager().m_AllowNameConversion)[make_pair(const_cast<char*>(type), const_cast<char*>(newName))]; + allowed.insert (const_cast<char*>(oldName)); +} + +void ClearTypeNameConversion() +{ + GetTranferNameConversionsManager().m_AllowTypeNameConversions->clear(); + GetTranferNameConversionsManager().m_AllowNameConversion->clear(); +} diff --git a/Runtime/Serialize/TransferFunctions/TransferNameConversions.h b/Runtime/Serialize/TransferFunctions/TransferNameConversions.h new file mode 100644 index 0000000..3082f8f --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/TransferNameConversions.h @@ -0,0 +1,56 @@ +#ifndef TRANSFERNAMECONVERSIONS_H +#define TRANSFERNAMECONVERSIONS_H + +#include "Runtime/Utilities/CStringHash.h" +#include "Runtime/Modules/ExportModules.h" + +using namespace std; + +struct smaller_cstring_pair : std::binary_function<std::pair<char*, char*>, std::pair<char*, char*>, std::size_t> +{ + bool operator () (pair<char*, char*> lhs, pair<char*, char*> rhs) const + { + int first = strcmp (lhs.first, rhs.first); + if (first != 0) + return first < 0; + else + return strcmp (lhs.second, rhs.second) < 0; + } +}; + +typedef std::multimap<char*, char*, smaller_cstring> AllowTypeNameConversions; +typedef std::map<std::pair<char*, char*>, set<char*, smaller_cstring>, smaller_cstring_pair> AllowNameConversion; + +class TranferNameConversionsManager +{ +public: + AllowTypeNameConversions* m_AllowTypeNameConversions; + AllowNameConversion* m_AllowNameConversion; + + TranferNameConversionsManager(); + ~TranferNameConversionsManager(); + + static TranferNameConversionsManager* s_Instance; + static void StaticInitialize(); + static void StaticDestroy(); +}; +inline TranferNameConversionsManager& GetTranferNameConversionsManager() { return *TranferNameConversionsManager::s_Instance; } + + +/// Allows type name conversion from oldTypeName to newTypeName(The passed strings will not be copied so you can only pass in constant strings) +/// (Useful for depracating types -> RegisterAllowTypeNameConversion ("UniqueIdentifier", "GUID");) +/// "UniqueIdentifier" can now be renamed to "GUID" and serialization will just work! +void RegisterAllowTypeNameConversion (const char* oldTypeName, const char* newTypeName); + +/// Allows name conversion from oldName to newName? (The passed strings will not be copied so you can only pass in constant strings) +/// (Useful for deprecating names -> m_NewPosition will now load from m_DeprecatedPosition in an old serialized file +/// RegisterAllowNameConversion (MyClass::GetClassStringStatic(), "m_DeprecatedPosition", "m_NewPosition"); +EXPORT_COREMODULE void RegisterAllowNameConversion (const char* type, const char* oldName, const char* newName); + +const AllowNameConversion::mapped_type* GetAllowedNameConversions (const char* type, const char* name); + +bool AllowTypeNameConversion (const UnityStr& oldType, const char* newTypeName); + +void ClearTypeNameConversion (); + +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLRead.cpp b/Runtime/Serialize/TransferFunctions/YAMLRead.cpp new file mode 100644 index 0000000..4a9ca42 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLRead.cpp @@ -0,0 +1,238 @@ +#include "UnityPrefix.h" +#include "YAMLRead.h" +#include "../FileCache.h" + +int YAMLRead::GetDataVersion () +{ + if (m_Versions.back() == -1) + { + yaml_node_t *node = m_CurrentNode; + int i = m_MetaParents.size(); + do + { + yaml_node_t *versionNode = GetValueForKey(node, "serializedVersion"); + if (versionNode) + { + Assert (versionNode->type == YAML_SCALAR_NODE); + sscanf ((char*)versionNode->data.scalar.value, "%d", &m_Versions.back()); + return m_Versions.back(); + } + // If "serializedVersion" is not found, look for "importerVersion" for backwards compatibility. + versionNode = GetValueForKey(node, "importerVersion"); + if (versionNode) + { + Assert (versionNode->type == YAML_SCALAR_NODE); + sscanf ((char*)versionNode->data.scalar.value, "%d", &m_Versions.back()); + return m_Versions.back(); + } + if (i>0) + node = m_MetaParents[--i]; + else + node = NULL; + } + while (node != NULL); + m_Versions.back() = 1; + } + return m_Versions.back(); +} + +yaml_node_t *YAMLRead::GetValueForKey (yaml_node_t* parentNode, const char* keystr) +{ + if (parentNode && parentNode->type == YAML_MAPPING_NODE) + { + // The code below does not handle empty yaml arrays. + if (parentNode->data.mapping.pairs.top == parentNode->data.mapping.pairs.start) + return NULL; + + yaml_node_pair_t* start; + if (m_CachedIndex < parentNode->data.mapping.pairs.top + && m_CachedIndex >= parentNode->data.mapping.pairs.start) + start = m_CachedIndex; + else + start = parentNode->data.mapping.pairs.start; + + yaml_node_pair_t* top = parentNode->data.mapping.pairs.top; + yaml_node_pair_t* i = start; + + do + { + yaml_node_pair_t* next = i+1; + if (next == top) + next = parentNode->data.mapping.pairs.start; + + yaml_node_t* key = yaml_document_get_node(m_ActiveDocument, i->key); + if (key == NULL) + { + // I've seen a crash bug report with no repro, indicating that this is happening. + // If you ever get this error and can repro it, let me know! jonas. + ErrorString ("YAML Node is NULL!\n"); + } + else + { + Assert (key->type == YAML_SCALAR_NODE); + + if (strcmp((char*)key->data.scalar.value, keystr) == 0) + { + m_CachedIndex = next; + return yaml_document_get_node(m_ActiveDocument, i->value); + } + } + i = next; + } + while (i != start); + } + return NULL; +} + + +void YAMLRead::Init(int flags, yaml_read_handler_t *handler, std::string *debugFileName, int debugLineCount) +{ + m_UserData = NULL; + m_CurrentVersion = 0; + m_Flags = flags; + m_CachedIndex = NULL; + m_ReadHandler = handler; + + yaml_parser_t parser; + + memset(&parser, 0, sizeof(parser)); + memset(&m_Document, 0, sizeof(m_Document)); + + if (!yaml_parser_initialize(&parser)) + { + ErrorString("Could not initialize yaml parser\n"); + return; + } + + yaml_parser_set_input(&parser, handler, this ); + yaml_parser_load(&parser, &m_Document); + + if (parser.error != YAML_NO_ERROR) + { + if (debugFileName != NULL) + { + ErrorStringMsg("Unable to parse file %s: [%s] at line %d\n", debugFileName->c_str(), parser.problem, debugLineCount + (int)parser.problem_mark.line); + } + else + { + ErrorStringMsg("Unable to parse YAML file: [%s] at line %d\n", parser.problem, debugLineCount + (int)parser.problem_mark.line); + } + } + + yaml_parser_delete(&parser); + + m_Versions.push_back(-1); + m_CurrentNode = yaml_document_get_root_node(&m_Document); + m_ActiveDocument = &m_Document; + m_DidReadLastProperty = false; +} + +YAMLRead::YAMLRead (yaml_document_t* yamlDocument, int flags) +: m_ReadHandler (NULL) +, m_ActiveDocument (yamlDocument) +, m_CurrentVersion (0) +, m_CachedIndex (0) +, m_DidReadLastProperty (false) +{ + m_Flags = flags; + memset(&m_Document, 0, sizeof(m_Document)); + m_Versions.push_back(-1); + m_CurrentNode = yaml_document_get_root_node(m_ActiveDocument); +} + + +int YAMLRead::YAMLReadCacheHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) +{ + YAMLRead *read = (YAMLRead*)data; + + if (read->m_ReadOffset + size > read->m_EndOffset) + size = read->m_EndOffset - read->m_ReadOffset; + + ReadFileCache (*(CacheReaderBase*)read->m_ReadData, buffer, read->m_ReadOffset, size); + read->m_ReadOffset += size; + *size_read = size; + + return true; +} + +int YAMLRead::YAMLReadStringHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read) +{ + YAMLRead *read = (YAMLRead*)data; + + if (read->m_ReadOffset + size > read->m_EndOffset) + size = read->m_EndOffset - read->m_ReadOffset; + + const char* readData = reinterpret_cast<const char*> (read->m_ReadData); + + memcpy (buffer, readData + read->m_ReadOffset, size); + read->m_ReadOffset += size; + *size_read = size; + + return true; +} + +YAMLRead::YAMLRead (const char* strBuffer, int size, int flags, std::string *debugFileName, int debugLineCount) +{ + m_ReadOffset = 0; + m_EndOffset = size; + m_ReadData = const_cast<char*> (strBuffer); + + Init (flags, YAMLReadStringHandler, debugFileName, debugLineCount); +} + +YAMLRead::YAMLRead (const CacheReaderBase *input, size_t readOffset, size_t endOffset, int flags, std::string *debugFileName, int debugLineCount) +{ + m_ReadOffset = readOffset; + m_EndOffset = endOffset; + m_ReadData = (void*)input; + + Init (flags, YAMLReadCacheHandler, debugFileName, debugLineCount); +} + +YAMLRead::~YAMLRead() +{ + yaml_document_delete(&m_Document); +} + +YAMLNode* YAMLRead::GetCurrentNode () +{ + return YAMLDocNodeToNode(m_ActiveDocument, m_CurrentNode); +} + +YAMLNode* YAMLRead::GetValueNodeForKey (const char* key) +{ + return YAMLDocNodeToNode (m_ActiveDocument, GetValueForKey (m_CurrentNode, key)); +} + +int YAMLRead::StringOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + string* theString = reinterpret_cast<string*> (data); + theString->append( (char *) buffer, size); + return 1; +} + +void YAMLRead::BeginMetaGroup (std::string name) +{ + m_MetaParents.push_back(m_CurrentNode); + m_CurrentNode = GetValueForKey(m_CurrentNode, name.c_str()); +} + +void YAMLRead::EndMetaGroup () +{ + m_CurrentNode = m_MetaParents.back(); + m_MetaParents.pop_back(); +} + +void YAMLRead::TransferTypelessData (unsigned size, void* data, int metaFlag) +{ + UnityStr dataString; + Transfer(dataString, "_typelessdata", metaFlag); + dataString.resize (size * 2); + HexStringToBytes (&dataString[0], size, data); +} + +bool YAMLRead::HasNode (const char* name) +{ + return GetValueForKey(m_CurrentNode, name) != NULL; +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLRead.h b/Runtime/Serialize/TransferFunctions/YAMLRead.h new file mode 100644 index 0000000..41e6148 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLRead.h @@ -0,0 +1,422 @@ +#ifndef YAMLREAD_H +#define YAMLREAD_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "TransferNameConversions.h" +#include "YAMLSerializeTraits.h" +#include "Editor/Src/Utility/YAMLNode.h" +#include "External/yaml/include/yaml.h" +#include "Runtime/Serialize/FloatStringConversion.h" +#include "Runtime/Utilities/TypeUtilities.h" + +class CacheReaderBase; +struct StreamingInfo; + +class YAMLRead : public TransferBase +{ +private: + + int m_CurrentVersion; + std::string m_CurrentType; + std::string m_NodeName; + bool m_DidReadLastProperty; + + yaml_document_t m_Document; + yaml_document_t* m_ActiveDocument; + std::vector<yaml_node_t*> m_MetaParents; + std::vector<int> m_Versions; + yaml_node_t* m_CurrentNode; + yaml_node_pair_t* m_CachedIndex; + yaml_read_handler_t *m_ReadHandler; + void* m_ReadData; + size_t m_ReadOffset; + size_t m_EndOffset; + + int GetDataVersion (); + yaml_node_t *GetValueForKey(yaml_node_t* parentNode, const char* keystr); + + static int YAMLReadCacheHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read); + static int YAMLReadStringHandler(void *data, unsigned char *buffer, size_t size, size_t *size_read); + + void Init(int flags, yaml_read_handler_t *handler, std::string *debugFileName, int debugLineCount); + +public: + + YAMLRead (const char* strBuffer, int size, int flags, std::string *debugFileName = NULL, int debugLineCount = 0); + YAMLRead (const CacheReaderBase *input, size_t readOffset, size_t endOffset, int flags, std::string *debugFileName = NULL, int debugLineCount = 0); + YAMLRead (yaml_document_t* yamlDocument, int flags); + ~YAMLRead(); + + void SetVersion (int version) { m_CurrentVersion = version; } + bool IsCurrentVersion () { return m_CurrentVersion == GetDataVersion(); } + bool IsOldVersion (int version) { return version == GetDataVersion(); } + bool IsVersionSmallerOrEqual (int version) { return version >= GetDataVersion(); } + + bool IsReading () { return true; } + bool IsReadingPPtr () { return true; } + bool IsReadingBackwardsCompatible() { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool DirectStringTransfer () { return true; } + bool AssetMetaDataOnly () { return m_Flags & kAssetMetaDataOnly; } + + bool IsSerializingForGameRelease () { return false; } + + bool DidReadLastProperty () { return m_DidReadLastProperty; } + bool DidReadLastPPtrProperty () { return m_DidReadLastProperty; } + + YAMLNode* GetCurrentNode (); + YAMLNode* GetValueNodeForKey (const char* key); + + static int StringOutputHandler(void *data, unsigned char *buffer, size_t size) ; + void BeginMetaGroup (std::string name); + void EndMetaGroup (); + + template<class T> + void Transfer (T& data, const char* name, int metaFlag = 0); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, int metaFlag = 0); + + void TransferTypeless (unsigned* value, const char* name, int metaFlag = 0) + { + Transfer(*value, name, metaFlag); + } + + bool HasNode (const char* name); + + void TransferTypelessData (unsigned size, void* data, int metaFlag = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferStringData (T& data); + + template<class T> + void TransferSTLStyleArray (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleMap (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleSet (T& data, int metaFlag = 0); + + template<class T> + void TransferPair (T& data, int metaFlag = 0, yaml_node_pair_t* pair = NULL); +}; + +template<> +inline void YAMLRead::TransferBasicData<bool> (bool& data) +{ + int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &i); + data = (i == 0); +} + +template<> +inline void YAMLRead::TransferBasicData<char> (char& data) +{ + //scanf on msvc does not support %hhd. read int instead + int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &i); + data = i; +} + +template<> +inline void YAMLRead::TransferBasicData<SInt8> (SInt8& data) +{ + TransferBasicData (reinterpret_cast<char&> (data)); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt8> (UInt8& data) +{ + //scanf on msvc does not support %hhu. read unsigned int instead + unsigned int i; + sscanf ((char*)m_CurrentNode->data.scalar.value, "%u", &i); + data = i; +} + +template<> +inline void YAMLRead::TransferBasicData<SInt16> (SInt16& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%hd", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt16> (UInt16& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%hu", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<SInt32> (SInt32& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%d", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt32> (UInt32& data) +{ + sscanf ((char*)m_CurrentNode->data.scalar.value, "%u", &data); +} + +template<> +inline void YAMLRead::TransferBasicData<SInt64> (SInt64& data) +{ + // msvc does not like %lld. Just read hex data directly. + Assert (strlen((char*)m_CurrentNode->data.scalar.value) == 16); + HexStringToBytes ((char*)m_CurrentNode->data.scalar.value, sizeof(SInt64), &data); +} + +template<> +inline void YAMLRead::TransferBasicData<UInt64> (UInt64& data) +{ + // msvc does not like %lld. Just read hex data directly. + Assert (strlen((char*)m_CurrentNode->data.scalar.value) == 16); + HexStringToBytes ((char*)m_CurrentNode->data.scalar.value, sizeof(UInt64), &data); +} + +template<> +inline void YAMLRead::TransferBasicData<double> (double& data) +{ + data = StringToDoubleAccurate((char*)m_CurrentNode->data.scalar.value); +} + +template<> +inline void YAMLRead::TransferBasicData<float> (float& data) +{ + data = StringToFloatAccurate ((char*)m_CurrentNode->data.scalar.value); +} + +template<class T> +inline void YAMLRead::TransferStringData (T& data) +{ + data = (char*)m_CurrentNode->data.scalar.value; +} + +template<class T> +void YAMLRead::Transfer (T& data, const char* _name, int metaFlag) +{ + m_DidReadLastProperty = false; + + if (metaFlag & kIgnoreInMetaFiles) + return; + + std::string name = YAMLSerializeTraits<T>::ParseName(_name, AssetMetaDataOnly()); + + yaml_node_t *parentNode = m_CurrentNode; + m_CurrentNode = GetValueForKey(parentNode, name.c_str()); + if (!m_CurrentNode) + { + const AllowNameConversion::mapped_type* nameConversion = GetAllowedNameConversions (m_CurrentType.c_str(), _name); + if (nameConversion) + { + for (AllowNameConversion::mapped_type::const_iterator i = nameConversion->begin(); i!=nameConversion->end();i++) + { + m_CurrentNode = GetValueForKey(parentNode, *i); + if (m_CurrentNode) + break; + } + } + if (!m_CurrentNode) + { + if (strcmp (_name, "Base") == 0) + { + if (parentNode && parentNode->type == YAML_MAPPING_NODE) + { + if (parentNode->data.mapping.pairs.start != parentNode->data.mapping.pairs.top) + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, parentNode->data.mapping.pairs.start->value); + } + } + else if (metaFlag & kTransferAsArrayEntryNameInMetaFiles) + { + YAMLSerializeTraits<T>::TransferStringToData (data, m_NodeName); + m_CurrentNode = parentNode; + return; + } + } + } + + std::string parentType = m_CurrentType; + m_CurrentType = SerializeTraits<T>::GetTypeString (&data); + + if (m_CurrentNode != NULL) + { + m_Versions.push_back(-1); + YAMLSerializeTraits<T>::Transfer (data, *this); + m_Versions.pop_back(); + m_DidReadLastProperty = true; + } + + m_CurrentNode = parentNode; + m_CurrentType = parentType; +} + +template<class T> +void YAMLRead::TransferWithTypeString (T& data, const char* name, const char*, int metaFlag) +{ + Transfer(data, name, metaFlag); +} + + +template<class T> +void YAMLRead::TransferSTLStyleArray (T& data, int /*metaFlag*/) +{ + yaml_node_t *parentNode = m_CurrentNode; + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + + switch (m_CurrentNode->type) + { + case YAML_SCALAR_NODE: + { +#if UNITY_BIG_ENDIAN +#error "Needs swapping to be implemented to work on big endian platforms!" +#endif + std::string str; + TransferStringData (str); + size_t byteLength = str.size() / 2; + size_t numElements = byteLength / sizeof(non_const_value_type); + SerializeTraits<T>::ResizeSTLStyleArray (data, numElements); + typename T::iterator dataIterator = data.begin (); + for (size_t i=0; i<numElements; i++) + { + HexStringToBytes (&str[i*2*sizeof(non_const_value_type)], sizeof(non_const_value_type), (void*)&*dataIterator); + ++dataIterator; + } + } + break; + + case YAML_SEQUENCE_NODE: + { + yaml_node_item_t* start = m_CurrentNode->data.sequence.items.start; + yaml_node_item_t* top = m_CurrentNode->data.sequence.items.top; + + SerializeTraits<T>::ResizeSTLStyleArray (data, top - start); + typename T::iterator dataIterator = data.begin (); + + for(yaml_node_item_t* i = start; i != top; i++) + { + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, *i); + YAMLSerializeTraits<non_const_value_type>::Transfer (*dataIterator, *this); + ++dataIterator; + } + } + break; + + // Some stupid old-style meta data writing code unnecessarily used mappings + // instead of sequences to encode arrays. So, we're able to read that as well. + case YAML_MAPPING_NODE: + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + + SerializeTraits<T>::ResizeSTLStyleArray (data, top - start); + typename T::iterator dataIterator = data.begin (); + + for(yaml_node_pair_t* i = start; i != top; i++) + { + yaml_node_t* key = yaml_document_get_node(m_ActiveDocument, i->key); + Assert (key->type == YAML_SCALAR_NODE); + + m_NodeName = (std::string)(char*)key->data.scalar.value; + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, i->value); + + YAMLSerializeTraits<non_const_value_type>::Transfer (*dataIterator, *this); + ++dataIterator; + } + } + break; + + default: + ErrorString("Unexpected node type."); + } + + m_CurrentNode = parentNode; +} + +template<class T> +void YAMLRead::TransferSTLStyleMap (T& data, int metaFlag) +{ + if (m_CurrentNode->type == YAML_MAPPING_NODE) + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + + data.clear(); + + yaml_node_t *parentNode = m_CurrentNode; + + for(yaml_node_pair_t* i = start; i != top; i++) + { + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + typedef typename non_const_value_type::first_type first_type; + non_const_value_type p; + + if (!YAMLSerializeTraits<first_type>::IsBasicType()) + { + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, i->value); + + YAMLSerializeTraits<non_const_value_type>::Transfer (p, *this); + } + else + TransferPair (p, metaFlag, i); + + data.insert (p); + } + m_CurrentNode = parentNode; + } +} + +template<class T> +void YAMLRead::TransferSTLStyleSet (T& data, int /*metaFlag*/) +{ + if (m_CurrentNode->type == YAML_SEQUENCE_NODE) + { + yaml_node_item_t* start = m_CurrentNode->data.sequence.items.start; + yaml_node_item_t* top = m_CurrentNode->data.sequence.items.top; + + data.clear(); + + yaml_node_t *parentNode = m_CurrentNode; + + for(yaml_node_item_t* i = start; i != top; i++) + { + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + non_const_value_type p; + + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, *i); + + YAMLSerializeTraits<non_const_value_type>::Transfer (p, *this); + + data.insert (p); + } + m_CurrentNode = parentNode; + } +} + +template<class T> +void YAMLRead::TransferPair (T& data, int /*metaFlag*/, yaml_node_pair_t* pair) +{ + typedef typename T::first_type first_type; + typedef typename T::second_type second_type; + + if (pair == NULL) + { + yaml_node_pair_t* start = m_CurrentNode->data.mapping.pairs.start; + yaml_node_pair_t* top = m_CurrentNode->data.mapping.pairs.top; + if (start == top) + return; + pair = start; + } + + yaml_node_t* parent = m_CurrentNode; + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, pair->key); + YAMLSerializeTraits<first_type>::Transfer (data.first, *this); + m_CurrentNode = yaml_document_get_node(m_ActiveDocument, pair->value); + YAMLSerializeTraits<second_type>::Transfer (data.second, *this); + m_CurrentNode = parent; +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp new file mode 100644 index 0000000..4dfffb8 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.cpp @@ -0,0 +1,139 @@ +#include "UnityPrefix.h" +#include "TransferNameConversions.h" +#include "YAMLRead.h" +#include "YAMLWrite.h" +#include "Runtime/BaseClasses/BaseObject.h" +#include "Editor/Src/GUIDPersistentManager.h" +#include "Runtime/Serialize/SerializedFile.h" +/* +@TODO: + +- Meta file for texture importer has additional unnecessary settings: + TextureImporter: + fileIDToRecycleName: {} + +- Add line offsets to SerializedFile ObjectInfo, so we can have proper line numbers in YAML errors. + +*/ + +// Text transfer of Unity References: +// If NeedsInstanceIDRemapping() is false or for null references: +// {instanceID: id} +// For local objects in same file: +// {fileID: id} +// For objects from other file with GUID: +// {fileID: id, guid: g, type: t} + +template<class T> +void TransferYAMLPtr (T& data, YAMLRead& transfer) +{ + SInt32 instanceID = 0; + if (!transfer.NeedsInstanceIDRemapping()) + { + transfer.Transfer (instanceID, "instanceID"); + data.SetInstanceID (instanceID); + } + else + { + bool allowLocalIdentifier = (transfer.GetFlags () & kYamlGlobalPPtrReference) == 0; + + LocalIdentifierInFileType fileID = 0; + TRANSFER (fileID); + + if (transfer.HasNode("guid")) + { + FileIdentifier id; + transfer.Transfer (id.guid, "guid"); + transfer.Transfer (id.type, "type"); + + id.Fix_3_5_BackwardsCompatibility (); + PersistentManager& pm = GetPersistentManager(); + SInt32 globalIndex = pm.InsertFileIdentifierInternal(id, true); + SerializedObjectIdentifier identifier (globalIndex, fileID); + + #if SUPPORT_INSTANCE_ID_REMAP_ON_LOAD + pm.ApplyInstanceIDRemap (identifier); + #endif + + instanceID = pm.SerializedObjectIdentifierToInstanceID (identifier); + } + else if (allowLocalIdentifier) + { + // local fileID + LocalSerializedObjectIdentifier identifier; + identifier.localIdentifierInFile = fileID; + identifier.localSerializedFileIndex = 0; + LocalSerializedObjectIdentifierToInstanceID (identifier, instanceID); + } + + data.SetInstanceID (instanceID); + } +} + +template<class T> +void TransferYAMLPtr (T& data, YAMLWrite& transfer) +{ + transfer.AddMetaFlag(kTransferUsingFlowMappingStyle); + SInt32 instanceID = data.GetInstanceID(); + if (!transfer.NeedsInstanceIDRemapping()) + transfer.Transfer (instanceID, "instanceID"); + else + { + // By default we allow writing self references that exclude guid & type. + // This way references inside of the file will never be lost even if the guid of the file changes + bool allowLocalIdentifier = (transfer.GetFlags () & kYamlGlobalPPtrReference) == 0; + if (allowLocalIdentifier) + { + LocalSerializedObjectIdentifier localIdentifier; + InstanceIDToLocalSerializedObjectIdentifier (instanceID, localIdentifier); + if (localIdentifier.localSerializedFileIndex == 0) + { + transfer.Transfer (localIdentifier.localIdentifierInFile, "fileID"); + return; + } + } + + GUIDPersistentManager& pm = GetGUIDPersistentManager(); + pm.Lock(); + SerializedObjectIdentifier identifier; + if (pm.InstanceIDToSerializedObjectIdentifier(instanceID, identifier)) + { + FileIdentifier id = pm.PathIDToFileIdentifierInternal(identifier.serializedFileIndex); + transfer.Transfer (identifier.localIdentifierInFile, "fileID"); + transfer.Transfer (id.guid, "guid"); + transfer.Transfer (id.type, "type"); + } + else + { + instanceID = 0; + transfer.Transfer (instanceID, "instanceID"); + } + pm.Unlock(); + } +} + +template<> +void TransferYAMLPPtr (PPtr<Object> &data, YAMLRead& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (PPtr<Object> &data, YAMLWrite& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, YAMLRead& transfer) { TransferYAMLPtr (data, transfer); } +template<> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, YAMLWrite& transfer) { TransferYAMLPtr (data, transfer); } + + +template<> +void YAMLSerializeTraits<UnityGUID>::Transfer (UnityGUID& data, YAMLRead& transfer) +{ + std::string str; + transfer.TransferStringData(str); + data = StringToGUID(str); +} + +template<> +void YAMLSerializeTraits<UnityGUID>::Transfer (UnityGUID& data, YAMLWrite& transfer) +{ + std::string str = GUIDToString(data); + transfer.TransferStringData(str); +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h new file mode 100644 index 0000000..04cf3aa --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLSerializeTraits.h @@ -0,0 +1,236 @@ +#ifndef YAMLSERIALIZETRAITS_H +#define YAMLSERIALIZETRAITS_H + +class Object; +template<class T> +class PPtr; +template<class T> +class ImmediatePtr; +struct UnityGUID; +class YAMLRead; +class YAMLWrite; + +template<class T> +class YAMLSerializeTraitsBase +{ + public: + inline static std::string ParseName (const char* _name, bool stripNames) + { + std::string name = _name; + + if (name == "Base") + name = SerializeTraits<T>::GetTypeString (NULL); + else if (stripNames) + { + if (name.rfind(".") != std::string::npos) + name = name.substr(name.rfind(".") + 1); + if (name.length() >= 3 && name.find("m_") == 0) + name = (char)tolower(name[2]) + name.substr(3); + } + return name; + } + + inline static bool ShouldSerializeArrayAsCompactString () + { + return false; + } + + inline static bool IsBasicType () + { + return false; + } + + template<class TransferFunction> inline + static void Transfer (T& data, TransferFunction& transfer) + { + SerializeTraits<T>::Transfer (data, transfer); + } + + inline static void TransferStringToData (T& /*data*/, std::string& /*str*/) + { + } +}; + +template<class T> +class YAMLSerializeTraits : public YAMLSerializeTraitsBase<T> {}; + +template<class T> +class YAMLSerializeTraitsForBasicType : public YAMLSerializeTraitsBase<T> +{ + public: + inline static bool ShouldSerializeArrayAsCompactString () + { + return true; + } + + inline static bool IsBasicType () + { + return true; + } +}; + +template<> +class YAMLSerializeTraits<UInt16> : public YAMLSerializeTraitsForBasicType<UInt16> {}; + +template<> +class YAMLSerializeTraits<SInt16> : public YAMLSerializeTraitsForBasicType<SInt16> {}; + +template<> +class YAMLSerializeTraits<UInt32> : public YAMLSerializeTraitsForBasicType<UInt32> {}; + +template<> +class YAMLSerializeTraits<SInt32> : public YAMLSerializeTraitsForBasicType<SInt32> {}; + +template<> +class YAMLSerializeTraits<UInt64> : public YAMLSerializeTraitsForBasicType<UInt64> {}; + +template<> +class YAMLSerializeTraits<SInt64> : public YAMLSerializeTraitsForBasicType<SInt64> {}; + +template<> +class YAMLSerializeTraits<UInt8> : public YAMLSerializeTraitsForBasicType<UInt8> {}; + +template<> +class YAMLSerializeTraits<SInt8> : public YAMLSerializeTraitsForBasicType<SInt8> {}; + +template<> +class YAMLSerializeTraits<char> : public YAMLSerializeTraitsForBasicType<char> {}; + +template<> +class YAMLSerializeTraits<bool> : public YAMLSerializeTraitsForBasicType<bool> {}; + +template<> +class YAMLSerializeTraits<UnityStr> : public YAMLSerializeTraitsBase<UnityStr > +{ +public: + + template<class TransferFunction> inline + static void Transfer (UnityStr& data, TransferFunction& transfer) + { + transfer.TransferStringData (data); + } + + inline static void TransferStringToData (UnityStr& data, std::string &str) + { + data = str.c_str(); + } + + inline static bool IsBasicType () + { + return true; + } +}; + +// Do not add this serialization function. All serialized strings should use UnityStr instead of std::string +//template<class Traits, class Allocator> +//class YAMLSerializeTraits<std::basic_string<char,Traits,Allocator> > : public YAMLSerializeTraitsBase<std::basic_string<char,Traits,Allocator> > + +template<class FirstClass, class SecondClass> +class YAMLSerializeTraits<std::pair<FirstClass, SecondClass> > : public YAMLSerializeTraitsBase<std::pair<FirstClass, SecondClass> > +{ + public: + + + template<class TransferFunction> inline + static void Transfer (std::pair<FirstClass, SecondClass>& data, TransferFunction& transfer) + { + if (YAMLSerializeTraits<FirstClass>::IsBasicType()) + transfer.TransferPair (data); + else + { + transfer.Transfer (data.first, "first"); + transfer.Transfer (data.second, "second"); + } + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class YAMLSerializeTraits<std::map<FirstClass, SecondClass, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::map<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::map<FirstClass, SecondClass, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + +template<class FirstClass, class SecondClass, class Compare, class Allocator> +class YAMLSerializeTraits<std::multimap<FirstClass, SecondClass, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::multimap<FirstClass, SecondClass, Compare, Allocator> > +{ + public: + + typedef std::multimap<FirstClass, SecondClass, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && SerializeTraits<FirstClass>::MightContainPPtr() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleMap (data); + } +}; + + +template<class T, class Compare, class Allocator> +class YAMLSerializeTraits<std::set<T, Compare, Allocator> > : public YAMLSerializeTraitsBase<std::set<T, Compare, Allocator> > +{ + public: + + typedef std::set<T, Compare, Allocator> value_type; + + template<class TransferFunction> inline + static void Transfer (value_type& data, TransferFunction& transfer) + { + AssertIf(transfer.IsRemapPPtrTransfer() && transfer.IsReadingPPtr()); + transfer.TransferSTLStyleSet (data); + } +}; + + +template<class TransferFunction> +void TransferYAMLPPtr (PPtr<Object> &data, TransferFunction& transfer); +template<class TransferFunction> +void TransferYAMLPPtr (ImmediatePtr<Object> &data, TransferFunction& transfer); + +template<class T> +class YAMLSerializeTraits<PPtr<T> > : public YAMLSerializeTraitsBase<PPtr<T> > +{ + public: + + template<class TransferFunction> inline + static void Transfer (PPtr<T>& data, TransferFunction& transfer) + { + TransferYAMLPPtr ((PPtr<Object>&)data, transfer); + } +}; + +template<class T> +class YAMLSerializeTraits<ImmediatePtr<T> > : public YAMLSerializeTraitsBase<ImmediatePtr<T> > +{ + public: + + template<class TransferFunction> inline + static void Transfer (ImmediatePtr<T>& data, TransferFunction& transfer) + { + TransferYAMLPPtr ((ImmediatePtr<Object>&)data, transfer); + } +}; + +template<> +class YAMLSerializeTraits<UnityGUID> : public YAMLSerializeTraitsBase<UnityGUID> +{ + public: + + template<class TransferFunction> + static void Transfer (UnityGUID& data, TransferFunction& transfer); + + inline static bool IsBasicType () + { + return true; + } +}; +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp b/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp new file mode 100644 index 0000000..ec2ea68 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWrite.cpp @@ -0,0 +1,183 @@ +#include "UnityPrefix.h" +#include "YAMLWrite.h" +#include "../CacheWrap.h" +#include <string> + +void YAMLWrite::TransferStringToCurrentNode (const char* str) +{ + if (m_Error) + return; + int node = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)str, strlen(str), YAML_ANY_SCALAR_STYLE); + if (node) + m_CurrentNode = node; + else + m_Error = true; +} + +int YAMLWrite::NewMapping () +{ + int node = yaml_document_add_mapping(&m_Document, NULL, (m_MetaFlags.back() & kTransferUsingFlowMappingStyle)? YAML_FLOW_MAPPING_STYLE : YAML_ANY_MAPPING_STYLE); + if (node == 0) + m_Error = true; + return node; +} + +int YAMLWrite::NewSequence () +{ + int node = yaml_document_add_sequence(&m_Document, NULL, YAML_ANY_SEQUENCE_STYLE); + if (node == 0) + m_Error = true; + return node; +} + +int YAMLWrite::GetNode () +{ + if (m_CurrentNode == -1) + m_CurrentNode = NewMapping(); + return m_CurrentNode; +} + +void YAMLWrite::AppendToNode(int parentNode, const char* keyStr, int valueNode) +{ + yaml_node_t* parent = yaml_document_get_node(&m_Document, parentNode); + switch (parent->type) + { + case YAML_MAPPING_NODE: + { + int keyNode = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)keyStr, strlen(keyStr), YAML_ANY_SCALAR_STYLE); + if (keyNode == 0) + m_Error = true; + yaml_document_append_mapping_pair(&m_Document, parentNode, keyNode, valueNode); + } + break; + + case YAML_SEQUENCE_NODE: + yaml_document_append_sequence_item(&m_Document, parentNode, valueNode); + break; + + default: + ErrorString("Unexpected node type."); + } +} + +int YAMLWrite::StringOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + string* theString = reinterpret_cast<string*> (data); + theString->append( (char *) buffer, size); + return 1; +} + +int YAMLWrite::CacheOutputHandler(void *data, unsigned char *buffer, size_t size) +{ + CachedWriter* cache = reinterpret_cast<CachedWriter*> (data); + cache->Write(buffer, size); + return 1; +} + +void YAMLWrite::OutputToHandler (yaml_write_handler_t *handler, void *data) +{ + yaml_node_t *root = yaml_document_get_root_node (&m_Document); + if (root->type == YAML_MAPPING_NODE && root->data.mapping.pairs.start != root->data.mapping.pairs.top) + { + yaml_emitter_t emitter; + memset(&emitter, 0, sizeof(emitter)); + + if (!yaml_emitter_initialize (&emitter)) + { + ErrorStringMsg ("Unable to write text file %s: yaml_emitter_initialize failed.", m_DebugFileName.c_str()); + return; + } + + yaml_emitter_set_output(&emitter, handler, data ); + yaml_emitter_dump(&emitter, &m_Document); + + if (emitter.error != YAML_NO_ERROR) + ErrorStringMsg ("Unable to write text file %s: %s.", m_DebugFileName.c_str(), emitter.problem); + + yaml_emitter_delete(&emitter); + } +} + +YAMLWrite::YAMLWrite (int flags, std::string *debugFileName) +{ + if (debugFileName) + m_DebugFileName = *debugFileName; + + memset(&m_Document, 0, sizeof(m_Document)); + + m_CurrentNode = -1; + m_Flags = flags; + m_Error = false; + m_MetaFlags.push_back (0); + m_UserData = NULL; + + if (!yaml_document_initialize(&m_Document, NULL, NULL, NULL, 1, 1)) + { + ErrorStringMsg ("Unable to write text file %s: yaml_document_initialize failed.", m_DebugFileName.c_str()); + m_Error = true; + } +} + +YAMLWrite::~YAMLWrite() +{ + yaml_document_delete(&m_Document); +} + +void YAMLWrite::OutputToCachedWriter (CachedWriter* writer) +{ + if (m_Error) + { + ErrorStringMsg ("Could not serialize text file %s because an error occured - we probably ran out of memory.", m_DebugFileName.c_str()); + return; + } + OutputToHandler (CacheOutputHandler, reinterpret_cast<void *>(writer)); +} + +void YAMLWrite::OutputToString (std::string& str) +{ + if (m_Error) + { + ErrorStringMsg ("Could not serialize text file %s because an error occured - we probably ran out of memory.", m_DebugFileName.c_str()); + return; + } + + OutputToHandler (StringOutputHandler, reinterpret_cast<void *>(&str)); +} + +void YAMLWrite::SetVersion (int version) +{ + char valueStr[256]; + snprintf(valueStr, 256, "%d", version); + int value = yaml_document_add_scalar(&m_Document, NULL, (yaml_char_t*)valueStr, strlen(valueStr), YAML_ANY_SCALAR_STYLE); + + AppendToNode (GetNode(), "serializedVersion", value); +} + +void YAMLWrite::BeginMetaGroup (std::string name) +{ + m_MetaParents.push_back (MetaParent()); + m_MetaParents.back().node = GetNode (); + m_MetaParents.back().name = name; + m_CurrentNode = NewMapping (); +} + +void YAMLWrite::EndMetaGroup () +{ + AppendToNode (m_MetaParents.back().node, m_MetaParents.back().name.c_str(), m_CurrentNode); + m_CurrentNode = m_MetaParents.back().node; + m_MetaParents.pop_back(); +} + +void YAMLWrite::StartSequence () +{ + m_CurrentNode = NewSequence(); +} + +void YAMLWrite::TransferTypelessData (unsigned size, void* data, int metaFlag) +{ + UnityStr dataString; + dataString.resize (size * 2); + BytesToHexString (data, size, &dataString[0]); + Transfer(dataString, "_typelessdata", metaFlag); +} + diff --git a/Runtime/Serialize/TransferFunctions/YAMLWrite.h b/Runtime/Serialize/TransferFunctions/YAMLWrite.h new file mode 100644 index 0000000..89c509c --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWrite.h @@ -0,0 +1,340 @@ +#ifndef YAMLWRITE_H +#define YAMLWRITE_H + +#include "Runtime/Serialize/TransferFunctions/TransferBase.h" +#include "YAMLSerializeTraits.h" +#include "Editor/Src/Utility/YAMLNode.h" +#include "External/yaml/include/yaml.h" +#include "Runtime/Serialize/FloatStringConversion.h" + +class CachedWriter; +struct YAMLConverterContext; + +class YAMLWrite : public TransferBase +{ +private: + + struct MetaParent + { + int node; + std::string name; + }; + + std::vector<MetaParent> m_MetaParents; + std::vector<int> m_MetaFlags; + yaml_document_t m_Document; + int m_CurrentNode; + bool m_Error; + std::string m_DebugFileName; + + void TransferStringToCurrentNode (const char* str); + int NewMapping (); + int NewSequence (); + int GetNode (); + void AppendToNode(int parentNode, const char* keyStr, int valueNode); + static int StringOutputHandler(void *data, unsigned char *buffer, size_t size); + static int CacheOutputHandler(void *data, unsigned char *buffer, size_t size); + + void OutputToHandler (yaml_write_handler_t *handler, void *data); + +public: + + YAMLWrite (int flags, std::string *debugFileName = NULL); + ~YAMLWrite(); + + void OutputToCachedWriter (CachedWriter* writer); + void OutputToString (std::string& str); + + // Sets the "version of the class currently transferred" + void SetVersion (int version); + + bool HasError () { return m_Error; } + bool IsWriting () { return true; } + bool IsWritingPPtr () { return true; } + bool NeedsInstanceIDRemapping () { return m_Flags & kNeedsInstanceIDRemapping; } + bool AssetMetaDataOnly () { return m_Flags & kAssetMetaDataOnly; } + bool IsSerializingForGameRelease () { return false; } + + void SetFlowMappingStyle (bool on); + + yaml_document_t* GetDocument () { return &m_Document; } + int GetCurrentNodeIndex () { return m_CurrentNode; } + + void PushMetaFlag (int flag) { m_MetaFlags.push_back(flag | m_MetaFlags.back());} + void PopMetaFlag () { m_MetaFlags.pop_back(); } + void AddMetaFlag(int mask) { m_MetaFlags.back() |= mask; } + + void BeginMetaGroup (std::string name); + void EndMetaGroup (); + void StartSequence (); + + template<class T> + void Transfer (T& data, const char* name, int metaFlag = 0); + template<class T> + void TransferWithTypeString (T& data, const char*, const char*, int metaFlag = 0); + + void TransferTypeless (unsigned* value, const char* name, int metaFlag = 0) + { + Transfer(*value, name, metaFlag); + } + + void TransferTypelessData (unsigned size, void* data, int metaFlag = 0); + + template<class T> + void TransferBasicData (T& data); + + template<class T> + void TransferPtr (bool, ReduceCopyData*){} + + template<class T> + void TransferStringData (T& data); + + template<class T> + void TransferSTLStyleArray (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleMap (T& data, int metaFlag = 0); + + template<class T> + void TransferSTLStyleSet (T& data, int metaFlag = 0); + + template<class T> + void TransferPair (T& data, int metaFlag = 0, int parent = -1); +}; + +template<> +inline void YAMLWrite::TransferBasicData<SInt64> (SInt64& data) +{ + char valueStr[17]; + BytesToHexString (&data, sizeof(SInt64), valueStr); + valueStr[16] = '\0'; + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt64> (UInt64& data) +{ + char valueStr[17]; + BytesToHexString (&data, sizeof(UInt64), valueStr); + valueStr[16] = '\0'; + TransferStringToCurrentNode (valueStr); +} + +// This are the definitions of std::numeric_limits<>::max_digits10, which we cannot use +// because it is only in the C++11 standard. +const int kMaxFloatDigits = std::floor(std::numeric_limits<float>::digits * 3010.0/10000.0 + 2); +const int kMaxDoubleDigits = std::floor(std::numeric_limits<double>::digits * 3010.0/10000.0 + 2); + +template<> +inline void YAMLWrite::TransferBasicData<float> (float& data) +{ + char valueStr[64]; + if (FloatToStringAccurate(data, valueStr, 64)) + TransferStringToCurrentNode (valueStr); + else + TransferStringToCurrentNode ("error"); +} + +template<> +inline void YAMLWrite::TransferBasicData<double> (double& data) +{ + char valueStr[64]; + if (DoubleToStringAccurate (data, valueStr, 64)) + TransferStringToCurrentNode (valueStr); + else + TransferStringToCurrentNode ("error"); +} + +template<> +inline void YAMLWrite::TransferBasicData<char> (char& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt8> (SInt8& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt8> (UInt8& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hhu", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt32> (SInt32& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%d", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt32> (UInt32& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%u", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<SInt16> (SInt16& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hd", data); + TransferStringToCurrentNode (valueStr); +} + +template<> +inline void YAMLWrite::TransferBasicData<UInt16> (UInt16& data) +{ + char valueStr[16]; + snprintf(valueStr, 16, "%hu", data); + TransferStringToCurrentNode (valueStr); +} + +template<class T> +inline void YAMLWrite::TransferStringData (T& data) +{ + TransferStringToCurrentNode (data.c_str()); +} + +template<class T> +void YAMLWrite::Transfer (T& data, const char* _name, int metaFlag) +{ + if (m_Error) + return; + + if (metaFlag & kIgnoreInMetaFiles) + return; + + std::string name = YAMLSerializeTraits<T>::ParseName(_name, AssetMetaDataOnly()); + + PushMetaFlag(0); + + int parent = GetNode(); + m_CurrentNode = -1; + + YAMLSerializeTraits<T>::Transfer (data, *this); + + if (m_CurrentNode != -1) + AppendToNode (parent, name.c_str(), m_CurrentNode); + + PopMetaFlag(); + + m_CurrentNode = parent; +} + +template<class T> +void YAMLWrite::TransferWithTypeString (T& data, const char* name, const char*, int metaFlag) +{ + Transfer(data, name, metaFlag); +} + + +template<class T> +void YAMLWrite::TransferSTLStyleArray (T& data, int metaFlag) +{ + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + if (YAMLSerializeTraits<non_const_value_type>::ShouldSerializeArrayAsCompactString()) + { +#if UNITY_BIG_ENDIAN +#error "Needs swapping to be implemented to work on big endian platforms!" +#endif + std::string str; + size_t numElements = data.size(); + size_t numBytes = numElements * sizeof(non_const_value_type); + str.resize (numBytes*2); + + typename T::iterator dataIterator = data.begin (); + for (size_t i=0; i<numElements; i++) + { + BytesToHexString ((void*)&*dataIterator, sizeof(non_const_value_type), &str[i*2*sizeof(non_const_value_type)]); + ++dataIterator; + } + + TransferStringData (str); + } + else + { + m_CurrentNode = NewSequence (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + while (i != end) + { + Transfer (*i, "data", metaFlag); + ++i; + } + } +} + +template<class T> +void YAMLWrite::TransferSTLStyleMap (T& data, int metaFlag) +{ + m_CurrentNode = NewMapping (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + + // maps value_type is: pair<const First, Second> + // So we have to write to maps non-const value type + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + typedef typename non_const_value_type::first_type first_type; + while (i != end) + { + non_const_value_type& p = (non_const_value_type&)(*i); + if (YAMLSerializeTraits<first_type>::IsBasicType()) + TransferPair (p, metaFlag, m_CurrentNode); + else + Transfer (p, "data", metaFlag); + i++; + } +} + +template<class T> +void YAMLWrite::TransferSTLStyleSet (T& data, int metaFlag) +{ + m_CurrentNode = NewSequence (); + + typename T::iterator i = data.begin (); + typename T::iterator end = data.end (); + + typedef typename NonConstContainerValueType<T>::value_type non_const_value_type; + while (i != end) + { + non_const_value_type& p = (non_const_value_type&)(*i); + Transfer (p, "data", metaFlag); + ++i; + } +} + +template<class T> +void YAMLWrite::TransferPair (T& data, int /*metaFlag*/, int parent) +{ + typedef typename T::first_type first_type; + typedef typename T::second_type second_type; + if (parent == -1) + parent = NewMapping (); + + m_CurrentNode = -1; + YAMLSerializeTraits<first_type>::Transfer (data.first, *this); + int key = m_CurrentNode; + + m_CurrentNode = -1; + YAMLSerializeTraits<second_type>::Transfer (data.second, *this); + + yaml_document_append_mapping_pair(&m_Document, parent, key, m_CurrentNode); + + m_CurrentNode = parent; +} +#endif diff --git a/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp b/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp new file mode 100644 index 0000000..b0596d6 --- /dev/null +++ b/Runtime/Serialize/TransferFunctions/YAMLWriteTests.cpp @@ -0,0 +1,58 @@ +#include "UnityPrefix.h" + +#ifdef ENABLE_UNIT_TESTS + +#include "YAMLWrite.h" +#include "Runtime/Testing/Testing.h" +#include <map> + +SUITE (YAMLWriteTests) +{ + struct Fixture + { + YAMLWrite instanceUnderTest; + Fixture () + : instanceUnderTest (0) {} + }; + + #define ROOT (yaml_document_get_root_node (instanceUnderTest.GetDocument ())) + #define FIRST_KEY_OF(node) (yaml_document_get_node (instanceUnderTest.GetDocument (), node->data.mapping.pairs.start->key)) + #define FIRST_VALUE_OF(node) (yaml_document_get_node (instanceUnderTest.GetDocument (), node->data.mapping.pairs.start->value)) + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithEmptyMap_ProducesMappingNode) + { + std::map<float, UnityStr> testMap; + instanceUnderTest.TransferSTLStyleMap (testMap); + CHECK (ROOT->type == YAML_MAPPING_NODE); + } + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithComplexKey_WritesDataChild) + { + // Arrange. + std::map<PPtr<Object>, UnityStr> testMap; + testMap[PPtr<Object> ()] = "bar"; + + // Act. + instanceUnderTest.TransferSTLStyleMap (testMap); + + // Assert. + CHECK (FIRST_KEY_OF (ROOT)->type == YAML_SCALAR_NODE); + CHECK (strcmp ((const char*) FIRST_KEY_OF (ROOT)->data.scalar.value, "data") == 0); + } + + TEST_FIXTURE (Fixture, TransferSTLStyleMap_WithBasicTypeKey_DoesNotWriteDataChild) + { + // Arrange. + std::map<int, UnityStr> testMap; + testMap[1234] = "bar"; + + // Act. + instanceUnderTest.TransferSTLStyleMap (testMap); + + // Assert. + CHECK (FIRST_KEY_OF (ROOT)->type == YAML_SCALAR_NODE); + CHECK (strcmp ((const char*) FIRST_KEY_OF (ROOT)->data.scalar.value, "1234") == 0); + } +} + +#endif // ENABLE_UNIT_TESTS |