Serialization.h

/***************************************************************************
 *                                                                         *
 *   Copyright (C) 2017-2025 Christian Schoenebeck                         *
 *                           <cuse@users.sourceforge.net>                  *
 *                                                                         *
 *   This library is part of libgig.                                       *
 *                                                                         *
 *   This library is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This library is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this library; if not, write to the Free Software           *
 *   Foundation, Inc., 59 Temple Place, Suite 330, Boston,                 *
 *   MA  02111-1307  USA                                                   *
 ***************************************************************************/
 
#ifndef LIBGIG_SERIALIZATION_H
#define LIBGIG_SERIALIZATION_H
 
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
 
#include <stdint.h>
#include <stdio.h>
#include <typeinfo>
#include <string>
#include <vector>
#include <map>
#include <set>
#include <time.h>
#include <stdarg.h>
#include <assert.h>
#include <functional>
#include <sstream>
 
#include "sysdef.h"
 
#ifndef __has_extension
# define __has_extension(x) 0
#endif
 
#ifndef HAS_BUILTIN_TYPE_TRAITS
# if __cplusplus >= 201103L
#  define HAS_BUILTIN_TYPE_TRAITS 1
# elif ( __has_extension(is_class) && __has_extension(is_enum) )
#  define HAS_BUILTIN_TYPE_TRAITS 1
# elif ( __GNUC__ > 4 || ( __GNUC__ == 4 && __GNUC_MINOR__ >= 3 ) )
#  define HAS_BUILTIN_TYPE_TRAITS 1
# elif _MSC_VER >= 1400 /* MS Visual C++ 8.0 (Visual Studio 2005) */
#  define HAS_BUILTIN_TYPE_TRAITS 1
# elif __INTEL_COMPILER >= 1100
#  define HAS_BUILTIN_TYPE_TRAITS 1
# else
#  define HAS_BUILTIN_TYPE_TRAITS 0
# endif
#endif
 
#if !HAS_BUILTIN_TYPE_TRAITS
# include <tr1/type_traits>
# define LIBGIG_IS_CLASS(type) std::tr1::__is_union_or_class<type>::value //NOTE: without compiler support we cannot distinguish union from class
#else
# define LIBGIG_IS_CLASS(type) __is_class(type)
#endif
 
namespace Serialization {
 
    // just symbol prototyping
    class DataType;
    class Object;
    class Member;
    class Archive;
    class ObjectPool;
    class Exception;
 
    typedef std::string String;
 
    template<class T>
    using Array = std::vector<T>;
 
    template<class T>
    using Set = std::set<T>;
 
    template<class T_key, class T_value>
    using Map = std::map<T_key,T_value>;
 
    typedef std::vector<uint8_t> RawData;
 
    typedef void* ID;
 
    typedef uint32_t Version;
 
    enum time_base_t {
        LOCAL_TIME, 
        UTC_TIME 
    };
 
    template<typename T>
    bool IsEnum(const T& /*data*/) {
        #if !HAS_BUILTIN_TYPE_TRAITS
        return std::tr1::is_enum<T>::value;
        #else
        return __is_enum(T);
        #endif
    }
 
    template<typename T>
    bool IsUnion(const T& /*data*/) {
        #if !HAS_BUILTIN_TYPE_TRAITS
        return false; // without compiler support we cannot distinguish union from class
        #else
        return __is_union(T);
        #endif
    }
 
    template<typename T>
    bool IsClass(const T& /*data*/) {
        #if !HAS_BUILTIN_TYPE_TRAITS
        return std::tr1::__is_union_or_class<T>::value; // without compiler support we cannot distinguish union from class
        #else
        return __is_class(T);
        #endif
    }
 
    /*template<typename T>
    bool IsTrivial(T data) {
        return __is_trivial(T);
    }*/
 
    /*template<typename T>
    bool IsPOD(T data) {
        return __is_pod(T);
    }*/
 
    /*template<typename T>
    bool IsArray(const T& data) {
        return false;
    }*/
 
    /*template<typename T>
    bool IsArray(const Array<T>& data) {
        return true;
    }*/
 
    template<typename T> inline
    String toString(const T& value) {
        return std::to_string(value);
    }
 
    template<typename T> inline
    String toString(T* ptr) {
        std::stringstream ss;
        ss << "0x" << std::hex << size_t(ptr);
        return ss.str();
    }
 
    template<> inline
    String toString(const String& value) {
        return value;
    }
 
    class UID {
    public:
        ID id; 
        size_t size; 
 
        bool isValid() const;
        operator bool() const { return isValid(); } 
        //bool operator()() const { return isValid(); }
        bool operator==(const UID& other) const { return id == other.id && size == other.size; }
        bool operator!=(const UID& other) const { return id != other.id || size != other.size; }
        bool operator<(const UID& other) const { return id < other.id || (id == other.id && size < other.size); }
        bool operator>(const UID& other) const { return id > other.id || (id == other.id && size > other.size); }
 
        template<typename T>
        static UID from(const T& obj) {
            return Resolver<T>::resolve(obj);
        }
 
    protected:
        // UID resolver for non-pointer types
        template<typename T>
        struct Resolver {
            static UID resolve(const T& obj) {
                const UID uid = { (ID) &obj, sizeof(obj) };
                return uid;
            }
        };
 
        // UID resolver for pointer types (of 1st degree)
        template<typename T>
        struct Resolver<T*> {
            static UID resolve(const T* const & obj) {
                const UID uid = { (ID) obj, sizeof(*obj) };
                return uid;
            }
        };
    };
 
    extern const UID NO_UID;
 
    typedef std::vector<UID> UIDChain;
 
#if LIBGIG_SERIALIZATION_INTERNAL
    // prototyping of private internal friend functions
    static String _encodePrimitiveValue(const Object& obj);
    static DataType _popDataTypeBlob(const char*& p, const char* end);
    static Member _popMemberBlob(const char*& p, const char* end);
    static Object _popObjectBlob(const char*& p, const char* end);
    static void _popPrimitiveValue(const char*& p, const char* end, Object& obj);
    static String _primitiveObjectValueToString(const Object& obj);
    //  |
    template<typename T>
    static T _primitiveObjectValueToNumber(const Object& obj);
#endif // LIBGIG_SERIALIZATION_INTERNAL
 
    class DataType {
    public:
        DataType();
        size_t size() const { return m_size; } 
        bool isValid() const;
        bool isPointer() const;
        bool isClass() const;
        bool isPrimitive() const;
        bool isString() const;
        bool isInteger() const;
        bool isReal() const;
        bool isBool() const;
        bool isEnum() const;
        bool isArray() const;
        bool isSet() const;
        bool isMap() const;
        bool isSigned() const;
        operator bool() const { return isValid(); } 
        //bool operator()() const { return isValid(); }
        bool operator==(const DataType& other) const;
        bool operator!=(const DataType& other) const;
        bool operator<(const DataType& other) const;
        bool operator>(const DataType& other) const;
        String asLongDescr() const;
        String baseTypeName() const;
        String customTypeName(bool demangle = false) const;
        String customTypeName2(bool demangle = false) const;
 
        template<typename T>
        static DataType dataTypeOf(const T& data) {
            const DataType type = Resolver<T>::resolve(data);
            registerNativeDataType(type, data);
            return type;
        }
 
        template<typename T>
        static DataType dataType() {
            T unused = T();
            const DataType type = Resolver<T>::resolve(unused);
            //FIXME: would create an endless recursion
            //registerNativeDataType(type, unused);
            return type;
        }
 
        template<typename T>
        static void registerNativeDataType();
 
        template<typename T>
        class NativeDataTypeRegistry {
        public:
            NativeDataTypeRegistry() {
                DataType::registerNativeDataType<T>();
            }
        };
 
    protected:
        DataType(bool isPointer, int size, String baseType,
                 String customType1 = "", String customType2 = "");
 
        String internalID() const;
        Object newInstance(Archive* archive) const;
 
        template<typename T, typename std::enable_if<std::is_default_constructible<T>::value, bool>::type = true>
        static void registerNativeDataType(const DataType& type, const T& nativeData);
 
        template<typename T, typename std::enable_if<!std::is_default_constructible<T>::value, bool>::type = true>
        static void registerNativeDataType(const DataType& type, const T& nativeData);
 
        template<typename T, bool T_isPointer>
        struct ResolverBase {
            static DataType resolve(const T& data) {
                // for pointers we must stick to the compile-time declared type
                // of pointer to prevent potential type mismatches caused by
                // RTTI in a polymorphic scenario
                const std::type_info& type = (T_isPointer) ? typeid(T) : typeid(data);
                const int sz = sizeof(data);
 
                // for primitive types we are using our own type names instead of
                // using std:::type_info::name(), because the precise output of the
                // latter may vary between compilers
                if (type == typeid(int8_t))   return DataType(T_isPointer, sz, "int8");
                if (type == typeid(uint8_t))  return DataType(T_isPointer, sz, "uint8");
                if (type == typeid(int16_t))  return DataType(T_isPointer, sz, "int16");
                if (type == typeid(uint16_t)) return DataType(T_isPointer, sz, "uint16");
                if (type == typeid(int32_t))  return DataType(T_isPointer, sz, "int32");
                if (type == typeid(uint32_t)) return DataType(T_isPointer, sz, "uint32");
                if (type == typeid(int64_t))  return DataType(T_isPointer, sz, "int64");
                if (type == typeid(uint64_t)) return DataType(T_isPointer, sz, "uint64");
                if (type == typeid(size_t)) {
                    if (sz == 1)  return DataType(T_isPointer, sz, "uint8");
                    if (sz == 2)  return DataType(T_isPointer, sz, "uint16");
                    if (sz == 4)  return DataType(T_isPointer, sz, "uint32");
                    if (sz == 8)  return DataType(T_isPointer, sz, "uint64");
                    else assert(false /* unknown size_t size */);
                }
                if (type == typeid(ssize_t)) {
                    if (sz == 1)  return DataType(T_isPointer, sz, "int8");
                    if (sz == 2)  return DataType(T_isPointer, sz, "int16");
                    if (sz == 4)  return DataType(T_isPointer, sz, "int32");
                    if (sz == 8)  return DataType(T_isPointer, sz, "int64");
                    else assert(false /* unknown ssize_t size */);
                }
                if (type == typeid(bool))     return DataType(T_isPointer, sz, "bool");
                if (type == typeid(float))    return DataType(T_isPointer, sz, "real32");
                if (type == typeid(double))   return DataType(T_isPointer, sz, "real64");
                if (type == typeid(String))   return DataType(T_isPointer, sz, "String");
 
                // for pointers we must stick to pointer's compile-time declared
                // type and not use RTTI for the pointed object, because the
                // latter would easily create a type mismatch conflict for
                // pointers and objects in a polymorphic scenario, because the
                // pointer might have been declared with a base class type,
                // while actually assigned objects to these pointers might be of
                // any other derived class and RTTI would resolve the latter
                const String rawCppName =
                    (T_isPointer) ? rawCppTypeName<T>()
                                  : rawCppTypeNameOf(data);
 
                if (IsEnum(data)) return DataType(T_isPointer, sz, "enum", rawCppName);
                if (IsUnion(data)) return DataType(T_isPointer, sz, "union", rawCppName);
                if (IsClass(data)) return DataType(T_isPointer, sz, "class", rawCppName);
 
                return DataType();
            }
        };
 
        // DataType resolver for non-pointer types
        template<typename T>
        struct Resolver : ResolverBase<T,false> {
            static DataType resolve(const T& data) {
                return ResolverBase<T,false>::resolve(data);
            }
        };
 
        // DataType resolver for pointer types (of 1st degree)
        template<typename T>
        struct Resolver<T*> : ResolverBase<T,true> {
            static DataType resolve(const T* const & data) {
                return ResolverBase<T,true>::resolve(*data);
            }
        };
 
        // DataType resolver for non-pointer Array<> container object types.
        template<typename T>
        struct Resolver<Array<T>> {
            static DataType resolve(const Array<T>& data) {
                const int sz = sizeof(data);
                T unused = T();
                return DataType(false, sz, "Array", rawCppTypeNameOf(unused));
            }
        };
 
        // DataType resolver for Array<> pointer types (of 1st degree).
        template<typename T>
        struct Resolver<Array<T>*> {
            static DataType resolve(const Array<T>*& data) {
                const int sz = sizeof(*data);
                T unused = T();
                return DataType(true, sz, "Array", rawCppTypeNameOf(unused));
            }
        };
 
        // DataType resolver for non-pointer Set<> container object types.
        template<typename T>
        struct Resolver<Set<T>> {
            static DataType resolve(const Set<T>& data) {
                const int sz = sizeof(data);
                T unused = T();
                return DataType(false, sz, "Set", rawCppTypeNameOf(unused));
            }
        };
 
        // DataType resolver for Set<> pointer types (of 1st degree).
        template<typename T>
        struct Resolver<Set<T>*> {
            static DataType resolve(const Set<T>*& data) {
                const int sz = sizeof(*data);
                T unused = T();
                return DataType(true, sz, "Set", rawCppTypeNameOf(unused));
            }
        };
 
        // DataType resolver for non-pointer Map<> container object types.
        template<typename T_key, typename T_value>
        struct Resolver<Map<T_key,T_value>> {
            static DataType resolve(const Map<T_key,T_value>& data) {
                const int sz = sizeof(data);
                T_key unused1 = T_key();
                T_value unused2 = T_value();
                return DataType(false, sz, "Map", rawCppTypeNameOf(unused1),
                                rawCppTypeNameOf(unused2));
            }
        };
 
        // DataType resolver for Map<> pointer types (of 1st degree).
        template<typename T_key, typename T_value>
        struct Resolver<Map<T_key,T_value>*> {
            static DataType resolve(const Map<T_key,T_value>*& data) {
                const int sz = sizeof(*data);
                T_key unused1 = T_key();
                T_value unused2 = T_value();
                return DataType(true, sz, "Map", rawCppTypeNameOf(unused1),
                                rawCppTypeNameOf(unused2));
            }
        };
 
        // for compile-time known types
        template<typename T>
        static String rawCppTypeName() {
            const std::type_info& type = typeid(T);
            #if defined _MSC_VER // Microsoft compiler ...
            String name = type.raw_name();
            #else // i.e. especially GCC and clang ...
            String name = type.name();
            #endif
            //while (!name.empty() && name[0] >= 0 && name[0] <= 9)
            //    name = name.substr(1);
            return name;
        }
 
        // for RTTI resolved pointer types
        template<typename T>
        static String rawCppTypeNameOf(const T* const & data) {
            return rawCppTypeName<T*>();
        }
 
        // for RTTI resolved non-pointer types
        template<typename T>
        static String rawCppTypeNameOf(const T& data) {
            const std::type_info& type = typeid(data);
            #if defined _MSC_VER // Microsoft compiler ...
            String name = type.raw_name();
            #else // i.e. especially GCC and clang ...
            String name = type.name();
            #endif
            //while (!name.empty() && name[0] >= 0 && name[0] <= 9)
            //    name = name.substr(1);
            return name;
        }
 
    private:
        String m_baseTypeName;
        String m_customTypeName;
        String m_customTypeName2;
        int m_size;
        bool m_isPointer;
        static std::map<String,std::function<Object(Archive*)>> m_allocFns;
 
#if LIBGIG_SERIALIZATION_INTERNAL
        friend DataType _popDataTypeBlob(const char*& p, const char* end);
#endif
        friend class Archive;
    };
 
    class Member {
    public:
        Member();
        UID uid() const;
        UID parentUID() const;
        String name() const;
        ssize_t offset() const;
        const DataType& type() const;
        bool isValid() const;
        operator bool() const { return isValid(); } 
        //bool operator()() const { return isValid(); }
        bool operator==(const Member& other) const;
        bool operator!=(const Member& other) const;
        bool operator<(const Member& other) const;
        bool operator>(const Member& other) const;
 
    protected:
        Member(String name, UID uid, ssize_t offset, DataType type, const Object& parent);
        friend class Archive;
 
    private:
        UID m_uid;
        ssize_t m_offset;
        String m_name;
        DataType m_type;
        UID m_parentUID;
 
#if LIBGIG_SERIALIZATION_INTERNAL
        friend Member _popMemberBlob(const char*& p, const char* end);
#endif
    };
 
    class Object {
    public:
        Object();
        Object(UIDChain uidChain, DataType type, const Object& parent);
 
        UID uid(int index = 0) const;
        const UIDChain& uidChain() const;
        UID parentUID() const;
        const DataType& type() const;
        const RawData& rawData() const;
        Version version() const;
        Version minVersion() const;
        bool isVersionCompatibleTo(const Object& other) const;
        std::vector<Member>& members();
        const std::vector<Member>& members() const;
        Member memberNamed(String name) const;
        Member memberByUID(const UID& uid) const;
        std::vector<Member> membersOfType(const DataType& type) const;
        int sequenceIndexOf(const Member& member) const;
        bool isValid() const;
        operator bool() const { return isValid(); } 
        //bool operator()() const { return isValid(); }
        bool operator==(const Object& other) const;
        bool operator!=(const Object& other) const;
        bool operator<(const Object& other) const;
        bool operator>(const Object& other) const;
        void setNativeValueFromString(const String& s);
 
    protected:
        void remove(const Member& member);
        void setVersion(Version v);
        void setMinVersion(Version v);
 
    private:
        DataType m_type;
        UIDChain m_uid;
        UID m_parentUID;
        Version m_version;
        Version m_minVersion;
        RawData m_data;
        std::vector<Member> m_members;
        std::function<void(Object& dstObj, const Object& srcObj, void* syncer)> m_sync;
 
#if LIBGIG_SERIALIZATION_INTERNAL
        friend String _encodePrimitiveValue(const Object& obj);
        friend Object _popObjectBlob(const char*& p, const char* end);
        friend void _popPrimitiveValue(const char*& p, const char* end, Object& obj);
        friend String _primitiveObjectValueToString(const Object& obj);
        // |
        template<typename T>
        friend T _primitiveObjectValueToNumber(const Object& obj);
#endif // LIBGIG_SERIALIZATION_INTERNAL
 
        friend class Archive;
    };
 
    class Archive {
    public:
        enum operation_t {
            OPERATION_NONE,        
            OPERATION_SERIALIZE,   
            OPERATION_DESERIALIZE  
        };
 
        Archive();
        Archive(const RawData& data);
        Archive(const uint8_t* data, size_t size);
        virtual ~Archive();
 
        template<typename T>
        void serialize(const T* obj) {
            m_operation = OPERATION_SERIALIZE;
            m_allObjects.clear();
            m_rawData.clear();
            m_root = UID::from(obj);
            const_cast<T*>(obj)->serialize(this);
            encode();
            m_operation = OPERATION_NONE;
        }
 
        template<typename T>
        void deserialize(T* obj) {
            Archive a;
            a.m_operation = m_operation = OPERATION_DESERIALIZE;
            obj->serialize(&a);
            a.m_root = UID::from(obj);
            Syncer s(a, *this);
            a.m_operation = m_operation = OPERATION_NONE;
        }
 
        template<typename T>
        void operator<<(const T& obj) {
            serialize(&obj);
        }
 
        template<typename T>
        void operator>>(T& obj) {
            deserialize(&obj);
        }
 
        const RawData& rawData();
        virtual String rawDataFormat() const;
 
        template<typename T_classType, typename T_memberType>
        void serializeMember(const T_classType& nativeObject, const T_memberType& nativeMember, const char* memberName) {
            const ssize_t offset =
                ((const uint8_t*)(const void*)&nativeMember) -
                ((const uint8_t*)(const void*)&nativeObject);
            const UIDChain uids = UIDChainResolver<T_memberType>(nativeMember);
            const DataType type = DataType::dataTypeOf(nativeMember);
            const UID parentUID = UID::from(nativeObject);
            Object& parent = m_allObjects[parentUID];
            if (!parent) {
                const UIDChain uids = UIDChainResolver<T_classType>(nativeObject);
                const DataType type = DataType::dataTypeOf(nativeObject);
                parent = Object(uids, type, Object());
            }
            const Member member(memberName, uids[0], offset, type, parent);
            parent.members().push_back(member);
            const Object obj(uids, type, parent);
            const bool bExistsAlready = m_allObjects.count(uids[0]);
            const bool isValidObject = obj;
            const Object& existingObj = m_allObjects[uids[0]];
            const bool bExistingObjectIsInvalid = !existingObj;
            if (isValidObject && (
                    !bExistsAlready || bExistingObjectIsInvalid || (
                        obj.parentUID() && !existingObj.parentUID()
                    )
               ))
            {
                m_allObjects[uids[0]] = obj;
                // recurse serialization for all members of this member
                // (only for struct/class types, noop for primitive types)
                SerializationRecursion<T_memberType>::serializeObject(this, nativeMember);
            }
        }
 
        template<typename T_classType, typename T_memberType>
        void serializeHeapMember(const T_classType& nativeObject, const T_memberType& heapMember, const char* memberName) {
            const ssize_t offset = -1; // used for all members on heap
            const UIDChain uids = UIDChainResolver<T_memberType>(heapMember);
            const DataType type = DataType::dataTypeOf(heapMember);
            const UID parentUID = UID::from(nativeObject);
            Object& parent = m_allObjects[parentUID];
            if (!parent) {
                const UIDChain uids = UIDChainResolver<T_classType>(nativeObject);
                const DataType type = DataType::dataTypeOf(nativeObject);
                parent = Object(uids, type, Object());
            }
            const Member member(memberName, uids[0], offset, type, parent);
            parent.members().push_back(member);
            const Object obj(uids, type, parent);
            const bool bExistsAlready = m_allObjects.count(uids[0]);
            const bool isValidObject = obj;
            const Object& existingObj = m_allObjects[uids[0]];
            const bool bExistingObjectIsInvalid = !existingObj;
            if (isValidObject && (
                    !bExistsAlready || bExistingObjectIsInvalid || (
                        obj.parentUID() && !existingObj.parentUID()
                    )
               ))
            {
                m_allObjects[uids[0]] = obj;
                // recurse serialization for all members of this member
                // (only for struct/class types, noop for primitive types)
                SerializationRecursion<T_memberType>::serializeObject(this, heapMember);
            }
        }
 
        template<typename T_objectType>
        void serializeAnonymousObject(const T_objectType& heapObject) {
            const UIDChain uids = UIDChainResolver<T_objectType>(heapObject);
            const DataType type = DataType::dataTypeOf(heapObject);
            const Object obj(uids, type, Object());
            const bool bExistsAlready = m_allObjects.count(uids[0]);
            const bool isValidObject = obj;
            const Object& existingObj = m_allObjects[uids[0]];
            const bool bExistingObjectIsInvalid = !existingObj;
            if (isValidObject && (
                    !bExistsAlready || bExistingObjectIsInvalid || (
                        obj.parentUID() && !existingObj.parentUID()
                    )
               ))
            {
                m_allObjects[uids[0]] = obj;
                // recurse serialization for all members of this object
                // (only for struct/class types, noop for primitive types)
                SerializationRecursion<T_objectType>::serializeObject(this, heapObject);
            }
        }
 
        template<typename T_classType>
        void setVersion(const T_classType& nativeObject, Version v) {
            const UID uid = UID::from(nativeObject);
            Object& obj = m_allObjects[uid];
            if (!obj) {
                const UIDChain uids = UIDChainResolver<T_classType>(nativeObject);
                const DataType type = DataType::dataTypeOf(nativeObject);
                obj = Object(uids, type, Object());
            }
            setVersion(obj, v);
        }
 
        template<typename T_classType>
        void setMinVersion(const T_classType& nativeObject, Version v) {
            const UID uid = UID::from(nativeObject);
            Object& obj = m_allObjects[uid];
            if (!obj) {
                const UIDChain uids = UIDChainResolver<T_classType>(nativeObject);
                const DataType type = DataType::dataTypeOf(nativeObject);
                obj = Object(uids, type, Object());
            }
            setMinVersion(obj, v);
        }
 
        virtual void decode(const RawData& data);
        virtual void decode(const uint8_t* data, size_t size);
        void clear();
        bool isModified() const;
        void removeMember(Object& parent, const Member& member);
        void remove(const Object& obj);
        Object& rootObject();
        Object& objectByUID(const UID& uid);
        Object& parentObjectOf(const Object& obj);
        Object& parentObjectOf(const Member& member);
        void setAutoValue(Object& object, String value);
        void setIntValue(Object& object, int64_t value);
        void setRealValue(Object& object, double value);
        void setBoolValue(Object& object, bool value);
        void setEnumValue(Object& object, uint64_t value);
        void setStringValue(Object& object, String value);
        String valueAsString(const Object& object);
        int64_t valueAsInt(const Object& object);
        double valueAsReal(const Object& object);
        bool valueAsBool(const Object& object);
        void setVersion(Object& object, Version v);
        void setMinVersion(Object& object, Version v);
        String name() const;
        void setName(String name);
        String comment() const;
        void setComment(String comment);
        time_t timeStampCreated() const;
        time_t timeStampModified() const;
        tm dateTimeCreated(time_base_t base = LOCAL_TIME) const;
        tm dateTimeModified(time_base_t base = LOCAL_TIME) const;
        operation_t operation() const;
 
    protected:
        // UID resolver for non-pointer types
        template<typename T>
        class UIDChainResolver {
        public:
            UIDChainResolver(const T& data) {
                m_uid.push_back(UID::from(data));
            }
 
            operator UIDChain() const { return m_uid; }
            UIDChain operator()() const { return m_uid; }
        private:
            UIDChain m_uid;
        };
 
        // UID resolver for pointer types (of 1st degree)
        template<typename T>
        class UIDChainResolver<T*> {
        public:
            UIDChainResolver(const T* const & data) {
                const UID uids[2] = {
                    { (ID) &data, sizeof(data) },
                    { (ID) data, sizeof(*data) }
                };
                m_uid.push_back(uids[0]);
                m_uid.push_back(uids[1]);
            }
 
            operator UIDChain() const { return m_uid; }
            UIDChain operator()() const { return m_uid; }
        private:
            UIDChain m_uid;
        };
 
        // SerializationRecursion for non-pointer class/struct types.
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl {
            static void serializeObject(Archive* archive, const T& obj) {
                const_cast<T&>(obj).serialize(archive);
            }
        };
 
        // SerializationRecursion for pointers (of 1st degree) to class/structs.
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl<T*,T_isRecursive> {
            static void serializeObject(Archive* archive, const T*& obj) {
                if (!obj) return;
                const_cast<T*&>(obj)->serialize(archive);
            }
        };
 
        // NOOP SerializationRecursion for primitive types.
        template<typename T>
        struct SerializationRecursionImpl<T,false> {
            static void serializeObject(Archive* /*archive*/, const T& /*obj*/) {}
        };
 
        // SerializationRecursion for pointers (of 1st degree) to primitive types.
        template<typename T>
        struct SerializationRecursionImpl<T*,false> {
            static void serializeObject(Archive* archive, const T* const & obj) {
                if (!obj) return;
                archive->serializeAnonymousObject(*obj);
            }
        };
 
        // NOOP SerializationRecursion for String objects.
        template<bool T_isRecursive>
        struct SerializationRecursionImpl<String,T_isRecursive> {
            static void serializeObject(Archive* archive, const String& obj) {}
        };
 
        // SerializationRecursion for String pointers (of 1st degree).
        template<bool T_isRecursive>
        struct SerializationRecursionImpl<String*,T_isRecursive> {
            static void serializeObject(Archive* archive, const String*& obj) {
                if (!obj) return;
                archive->serializeAnonymousObject(*obj);
            }
        };
 
        // SerializationRecursion for Array<> objects.
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl<Array<T>,T_isRecursive> {
            static void serializeObject(Archive* archive, const Array<T>& obj) {
                const UIDChain uids = UIDChainResolver<Array<T>>(obj);
                const Object& object = archive->objectByUID(uids[0]);
                if (archive->operation() == OPERATION_SERIALIZE) {
                    for (size_t i = 0; i < obj.size(); ++i) {
                        archive->serializeHeapMember(
                            obj, obj[i], ("[" + toString(i) + "]").c_str()
                        );
                    }
                } else {
                    const_cast<Object&>(object).m_sync =
                        [&obj,archive](Object& dstObj, const Object& srcObj,
                                       void* syncer)
                    {
                        const size_t n = srcObj.members().size();
                        const_cast<Array<T>&>(obj).resize(n);
                        for (size_t i = 0; i < n; ++i) {
                            archive->serializeHeapMember(
                                obj, obj[i], ("[" + toString(i) + "]").c_str()
                            );
                        }
                        // updating dstObj required as serializeHeapMember()
                        // replaced the original object by a new one
                        dstObj = archive->objectByUID(dstObj.uid());
                        for (size_t i = 0; i < n; ++i) {
                            String name = "[" + toString(i) + "]";
                            Member srcMember = srcObj.memberNamed(name);
                            Member dstMember = dstObj.memberNamed(name);
                            ((Syncer*)syncer)->syncMember(dstMember, srcMember);
                        }
                    };
                }
            }
        };
 
        // SerializationRecursion for Array<> pointers (of 1st degree).
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl<Array<T>*,T_isRecursive> {
            static void serializeObject(Archive* archive, const Array<T>*& obj) {
                if (!obj) return;
                SerializationRecursionImpl<Array<T>,T_isRecursive>::serializeObject(
                    archive, *obj
                );
            }
        };
 
        // SerializationRecursion for Set<> objects.
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl<Set<T>,T_isRecursive> {
            static void serializeObject(Archive* archive, const Set<T>& obj) {
                const UIDChain uids = UIDChainResolver<Set<T>>(obj);
                const Object& object = archive->objectByUID(uids[0]);
                if (archive->operation() == OPERATION_SERIALIZE) {
                    for (const T& key : obj) {
                        archive->serializeHeapMember(
                            obj, key, ("[" + toString(key) + "]").c_str()
                        );
                    }
                } else {
                    const_cast<Object&>(object).m_sync =
                        [&obj,archive](Object& dstObj, const Object& srcObj,
                                       void* syncer)
                    {
                        const size_t n = srcObj.members().size();
                        const_cast<Set<T>&>(obj).clear();
                        for (size_t i = 0; i < n; ++i) {
                            const Member& member = srcObj.members()[i];
                            String name = member.name();
                            // strip brackets from name
                            if (name.length() < 2 || name[0] != '[' ||
                                *name.rbegin() != ']') continue;
                            name = name.substr(1, name.length() - 2);
                            T key;
                            const UIDChain uids = UIDChainResolver<T>(key);
                            const DataType type = DataType::dataTypeOf(key);
                            Object tmpObj(uids, type, Object());
                            // set the value of "key" variable by abstraction API
                            tmpObj.setNativeValueFromString(name);
                            // only for keys of pointer type: translation of
                            // memory address from source archive to destination
                            // archive required, NOOP for all other data types
                            ((Syncer*)syncer)->translateKey(key);
                            // insert (translated) key into set
                            const_cast<Set<T>&>(obj).insert(key);
                        }
                        // continue serialization recursion
                        for (const T& key : obj) {
                            archive->serializeHeapMember(
                                obj, key, ("[" + toString(key) + "]").c_str()
                            );
                        }
                        // updating dstObj required as serializeHeapMember()
                        // replaced the original object by a new one
                        dstObj = archive->objectByUID(dstObj.uid());
                    };
                }
            }
        };
 
        // SerializationRecursion for Set<> pointers (of 1st degree).
        template<typename T, bool T_isRecursive>
        struct SerializationRecursionImpl<Set<T>*,T_isRecursive> {
            static void serializeObject(Archive* archive, const Set<T>*& obj) {
                if (!obj) return;
                SerializationRecursionImpl<Set<T>,T_isRecursive>::serializeObject(
                    archive, *obj
                );
            }
        };
 
        // SerializationRecursion for Map<> objects.
        template<typename T_key, typename T_value, bool T_isRecursive>
        struct SerializationRecursionImpl<Map<T_key,T_value>,T_isRecursive> {
            static void serializeObject(Archive* archive, const Map<T_key,T_value>& obj) {
                const UIDChain uids = UIDChainResolver<Map<T_key,T_value>>(obj);
                const Object& object = archive->objectByUID(uids[0]);
                if (archive->operation() == OPERATION_SERIALIZE) {
                    for (const auto& it : obj) {
                        // recurse serialization for map's key ...
                        SerializationRecursion<T_key>::serializeObject(archive, it.first);
                        // ... and map's value
                        archive->serializeHeapMember(
                            obj, it.second, ("[" + toString(it.first) + "]").c_str()
                        );
                    }
                } else {
                    const_cast<Object&>(object).m_sync =
                        [&obj,archive](Object& dstObj, const Object& srcObj,
                                       void* syncer)
                    {
                        const size_t n = srcObj.members().size();
                        const_cast<Map<T_key,T_value>&>(obj).clear();
                        // translation is neutral except for keys of pointer type (see comments below)
                        std::map<String,String> memberNameTranslation;
                        for (size_t i = 0; i < n; ++i) {
                            const Member& member = srcObj.members()[i];
                            String name = member.name();
                            // strip brackets from name
                            if (name.length() < 2 || name[0] != '[' ||
                                *name.rbegin() != ']') continue;
                            name = name.substr(1, name.length() - 2);
                            T_key srcKey;
                            const UIDChain uids = UIDChainResolver<T_key>(srcKey);
                            const DataType type = DataType::dataTypeOf(srcKey);
                            Object tmpObj(uids, type, Object());
                            // set the value of "srcKey" variable by abstraction API
                            tmpObj.setNativeValueFromString(name);
                            T_key dstKey = srcKey;
                            // only for keys of pointer type: translation of
                            // memory address from source archive to destination
                            // archive required, NOOP for all other data types
                            ((Syncer*)syncer)->translateKey(dstKey);
                            memberNameTranslation[member.name()] = "[" + toString(dstKey) + "]";
                            // insert (translated) key into the destination map
                            const_cast<Map<T_key,T_value>&>(obj)[dstKey] = T_value();
                        }
                        // continue serialization recursion ...
                        for (const auto& it : obj) {
                            // ... for map's key ...
                            SerializationRecursion<T_key>::serializeObject(archive, it.first);
                            // ... and map's value
                            archive->serializeHeapMember(
                                obj, it.second, ("[" + toString(it.first) + "]").c_str()
                            );
                        }
                        // updating dstObj required as serializeHeapMember()
                        // replaced the original object by a new one
                        dstObj = archive->objectByUID(dstObj.uid());
                        // sync map's values
                        for (size_t i = 0; i < n; ++i) {
                            Member srcMember = srcObj.members()[i];
                            Member dstMember = dstObj.memberNamed(
                                memberNameTranslation[srcMember.name()]
                            );
                            ((Syncer*)syncer)->syncMember(dstMember, srcMember);
                        }
                    };
                }
            }
        };
 
        // SerializationRecursion for Map<> pointers (of 1st degree).
        template<typename T_key, typename T_value, bool T_isRecursive>
        struct SerializationRecursionImpl<Map<T_key,T_value>*,T_isRecursive> {
            static void serializeObject(Archive* archive, const Map<T_key,T_value>*& obj) {
                if (!obj) return;
                SerializationRecursionImpl<Map<T_key,T_value>,T_isRecursive>::serializeObject(
                    archive, *obj
                );
            }
        };
 
        // Automatically handles recursion for class/struct types, while ignoring all primitive types.
        template<typename T>
        struct SerializationRecursion : SerializationRecursionImpl<T, LIBGIG_IS_CLASS(T)> {
        };
 
        class ObjectPool : public std::map<UID,Object> {
        public:
            // prevent passing obvious invalid UID values from creating a new pair entry
            Object& operator[](const UID& k) {
                static Object invalid;
                if (!k.isValid()) {
                    invalid = Object();
                    return invalid;
                }
                return std::map<UID,Object>::operator[](k);
            }
        };
 
        friend String _encode(const ObjectPool& objects);
 
    private:
        String _encodeRootBlob();
        void _popRootBlob(const char*& p, const char* end);
        void _popObjectsBlob(const char*& p, const char* end);
 
    protected:
        class Syncer {
        public:
            Syncer(Archive& dst, Archive& src);
            void syncObject(const Object& dst, const Object& src);
            void syncPrimitive(const Object& dst, const Object& src);
            void syncString(const Object& dst, const Object& src);
            void syncArray(const Object& dst, const Object& src);
            void syncSet(const Object& dst, const Object& src);
            void syncMap(const Object& dst, const Object& src);
            void syncPointer(const Object& dst, const Object& src);
            void syncMember(const Member& dstMember, const Member& srcMember);
 
            // Address translation for keys of pointer type.
            //
            // Translates memory address from source archive, to memory address
            // of destination archive. For "strong" pointers memory is allocated
            // and the pointed data is synced with data from source archive.
            template<typename T>
            void translateKey(T*& key) {
                const UIDChain uids = UIDChainResolver<T*>(key);
                assert(uids.size() == 2);
                if (!uids[1])
                    return; // NULL pointer on source side
                const Object& pointedSrcObject = m_src.m_allObjects[uids[1]];
                assert(pointedSrcObject);
                std::map<UID,UID>::iterator uidRelation = m_counterparts.find(uids[1]);
                if (pointedSrcObject.parentUID() || uidRelation != m_counterparts.end()) { // "weak" pointer to object ...
                    assert(uidRelation != m_counterparts.end());
                    key = (T*) uidRelation->second.id;
                } else { // "strong" pointer to object, allocation required ...
                    assert(pointedSrcObject.type());
                    Object pointedDstObject = pointedSrcObject.type().newInstance(&m_dst);
                    assert(pointedDstObject);
                    m_dst.m_allObjects[pointedDstObject.uid()] = pointedDstObject;
                    key = (T*) pointedDstObject.uid().id;
 
                    syncObject(pointedDstObject, pointedSrcObject);
                }
            }
 
            // NOOP, no key translation required for any other type
            template<typename T>
            void translateKey(T& key) {}
        protected:
            static Member dstMemberMatching(const Object& dstObj, const Object& srcObj, const Member& srcMember);
        private:
            Archive& m_dst;
            Archive& m_src;
            std::map<UID,UID> m_counterparts;
        };
 
        virtual void encode();
 
        ObjectPool m_allObjects;
        operation_t m_operation;
        UID m_root;
        RawData m_rawData;
        bool m_isModified;
        String m_name;
        String m_comment;
        time_t m_timeCreated;
        time_t m_timeModified;
 
        friend class DataType;
    };
 
    // Manual registration of native data types by application.
    template<typename T>
    void DataType::registerNativeDataType() {
        static_assert(
            std::is_default_constructible<T>::value,
            "missing default constructor for type: registration of data "
            "types is only needed for creating instances of that type at "
            "runtime by this framework's reflection API; which in turn "
            "requires the type to provide a default constructor."
        );
        const DataType staticType = DataType::dataType<T>();
        assert(staticType);
        if (m_allocFns.find(staticType.internalID()) != m_allocFns.end()) return;
 
        // register lambda function for allocating a new instance of this type
        m_allocFns[staticType.internalID()] = [](Archive* archive) -> Object {
            T* instance = new T;
            Archive::SerializationRecursion<T>::serializeObject(archive, *instance);
            const UIDChain uids = Archive::UIDChainResolver<T>(*instance);
            Object& obj = archive->objectByUID(uids[0]);
            if (obj) return obj;
            const DataType type = DataType::dataTypeOf(*instance);
            return Object(uids, type, Object());
        };
    }
 
    // Automatic self-registration of native data types by this framework.
    // (SFINAE variant for types WITH default constructor)
    template<typename T, typename std::enable_if<std::is_default_constructible<T>::value, bool>::type>
    void DataType::registerNativeDataType(const DataType& rttiType, const T& nativeData) {
        if (!rttiType) return;
        if (m_allocFns.find(rttiType.internalID()) != m_allocFns.end()) return;
 
        // Ensure compile-time native data type 'T' matches passed runtime
        // native data type 'nativeData', because we can only offer to create
        // new instances of native data types by our reflection API if this
        // framework is aware about the data type already. Usually this happens
        // automatically without applications having to do anything. But
        // this only works for types this framework gets in touch with at
        // compile-time.
        //
        // Root problem of all of this: ATM there is no way in C++ to create new
        // instances of a type by using RTTI (typeid(), std::type_info, etc.).
        // If there is a way in a future C++ version then we can get rid of
        // manual type registration altogether. Bur for now we must use new T,
        // which is a compile-time only construct.
        //
        // See the discussion in DataType::registerNativeDataType() for an
        // example where manual type registration would be required.
        const DataType staticType = DataType::dataType<T>();
        if (staticType != rttiType) {
            fprintf(stderr, "Serialization Failure: run-time data type '%s' does not match compile-time data type '%s'!\n",
                    rttiType.asLongDescr().c_str(),
                    staticType.asLongDescr().c_str());
            assert(false && "You may need to explicitly register this run-time "
                            "data type by either calling "
                            "DataType::registerNativeType<T>() or using class "
                            "NativeDataTypeRegistry");
        }
 
        // register lambda function for allocating a new instance of this type
        m_allocFns[rttiType.internalID()] = [](Archive* archive) -> Object {
            T* instance = new T;
            Archive::SerializationRecursion<T>::serializeObject(archive, *instance);
            const UIDChain uids = Archive::UIDChainResolver<T>(*instance);
            Object& obj = archive->objectByUID(uids[0]);
            if (obj) return obj;
            const DataType type = DataType::dataTypeOf(*instance);
            return Object(uids, type, Object());
        };
    }
 
    // Automatic self-registration of native data types by this framework.
    // (SFINAE variant for types WITHOUT default constructor)
    template<typename T, typename std::enable_if<!std::is_default_constructible<T>::value, bool>::type>
    void DataType::registerNativeDataType(const DataType& type, const T& nativeData) {
        if (!type) return;
        if (m_allocFns.find(type.internalID()) != m_allocFns.end()) return;
        m_allocFns[type.internalID()] = [](Archive*) -> Object {
            assert(false && "instance not possible: native data type does not have default constructor");
            return Object();
        };
    }
 
    class Exception {
        public:
            String Message;
 
            Exception(String format, ...);
            Exception(String format, va_list arg);
            void PrintMessage() const;
            virtual ~Exception() {}
 
        protected:
            Exception();
            static String assemble(String format, va_list arg);
    };
 
} // namespace Serialization
 
#endif // LIBGIG_SERIALIZATION_H