diff --git a/src/MAT.jl b/src/MAT.jl index e0a4b6a..010c7cf 100644 --- a/src/MAT.jl +++ b/src/MAT.jl @@ -204,31 +204,43 @@ Write a dictionary containing variable names as keys and values as values to a Matlab file, opening and closing it automatically. """ function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::Bool = false, version::String ="v7.3") where {S, T} - if version == "v4" - file = open(filename, "w") - m = MAT_v4.Matlabv4File(file, false) - _write_dict(m, dict) - elseif version == "v7.3" - file = matopen(filename, "w"; compress = compress) - _write_dict(file, dict) - else - error("writing for \"$(version)\" is not supported") + file = nothing + try + if version == "v4" + file = open(filename, "w") + file = MAT_v4.Matlabv4File(file, false) + _write_dict(file, dict) + elseif version == "v7.3" + file = matopen(filename, "w"; compress = compress) + _write_dict(file, dict) + else + error("writing for \"$(version)\" is not supported") + end + finally + if file !== nothing + close(file) + end end end function _write_dict(fileio, dict::AbstractDict) - try - for (k, v) in dict - local kstring - try - kstring = ascii(convert(String, k)) - catch x - error("matwrite requires a Dict with ASCII keys") - end - write(fileio, kstring, v) + + for (k, v) in dict + local kstring + try + kstring = ascii(convert(String, k)) + catch x + error("matwrite requires a Dict with ASCII keys") + end + write(fileio, kstring, v) + end + + if hasproperty(fileio, :subsystem) && fileio.subsystem !== nothing + # will always be nothing for MATv4 so we can ignore that case + subsys_data = MAT_subsys.set_subsystem_data!(fileio.subsystem) + if subsys_data !== nothing + MAT_HDF5.write_subsys(fileio, subsys_data) end - finally - close(fileio) end end diff --git a/src/MAT_HDF5.jl b/src/MAT_HDF5.jl index 66f7361..bb8f849 100644 --- a/src/MAT_HDF5.jl +++ b/src/MAT_HDF5.jl @@ -36,7 +36,7 @@ import HDF5: Reference import Dates import Tables import PooledArrays: PooledArray -import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable +import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable, EmptyStruct const HDF5Parent = Union{HDF5.File, HDF5.Group} const HDF5BitsOrBool = Union{HDF5.BitsType,Bool} @@ -130,10 +130,12 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo close(g) end subsys_refs = "#subsystem#" - if haskey(fid.plain, subsys_refs) + if rd && haskey(fid.plain, subsys_refs) fid.subsystem.table_type = table subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem) MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator) + elseif wr + MAT_subsys.init_save!(fid.subsystem) end fid end @@ -464,7 +466,7 @@ function _normalize_arr(x) end # Write a scalar or array -function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}) where {T<:HDF5BitsOrBool} +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}, ) where {T<:HDF5BitsOrBool} data = _normalize_arr(data) if isempty(data) m_writeempty(parent, name, data) @@ -541,14 +543,19 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, c::Abs end # Write cell arrays -function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}) where T +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}, object_decode::UInt32=UInt32(0)) where T data = _normalize_arr(data) refs = _write_references!(mfile, parent, data) # Write the references as the chosen variable cset, ctype = create_dataset(parent, name, refs) try write_dataset(cset, ctype, refs) - write_attribute(cset, name_type_attr_matlab, "cell") + if object_decode == UInt32(3) + write_attribute(cset, object_decode_attr_matlab, object_decode) + write_attribute(cset, name_type_attr_matlab, "FileWrapper__") + else + write_attribute(cset, name_type_attr_matlab, "cell") + end finally close(ctype) close(cset) @@ -680,8 +687,34 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M end end +# Write empty (zero-dimensional) structs with no fields +function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s::EmptyStruct) + dset, dtype = create_dataset(parent, name, s.dims) + try + write_attribute(dset, empty_attr_matlab, 0x01) + write_attribute(dset, name_type_attr_matlab, "struct") + write_dataset(dset, dtype, s.dims) + finally + close(dtype); close(dset) + end +end + # Write a struct from arrays of keys and values function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, k::Vector{String}, v::Vector) + if length(k) == 0 + # empty struct + adata = UInt64[1, 1] + dset, dtype = create_dataset(parent, name, adata) + try + write_attribute(dset, empty_attr_matlab, 0x01) + write_attribute(dset, name_type_attr_matlab, "struct") + write_dataset(dset, dtype, adata) + finally + close(dtype); close(dset) + end + return + end + g = create_group(parent, name) try write_attribute(g, name_type_attr_matlab, "struct") @@ -719,11 +752,35 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::D end function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::MatlabOpaque) - error("writing of MatlabOpaque types is not yet supported") + if obj.class == "FileWrapper__" + m_write(mfile, parent, name, obj["__filewrapper__"], UInt32(3)) + return + end + + metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj) + dset, dtype = create_dataset(parent, name, metadata) + try + write_dataset(dset, dtype, metadata) + write_attribute(dset, name_type_attr_matlab, obj.class) + write_attribute(dset, object_type_attr_matlab, UInt32(3)) + finally + close(dset) + close(dtype) + end end function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::AbstractArray{MatlabOpaque}) - error("writing of MatlabOpaque types is not yet supported") + metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj) + dset, dtype = create_dataset(parent, name, metadata) + try + # TODO: Handle empty array case + write_dataset(dset, dtype, metadata) + write_attribute(dset, name_type_attr_matlab, first(obj).class) + write_attribute(dset, object_type_attr_matlab, UInt32(3)) + finally + close(dset) + close(dtype) + end end function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::PooledArray) @@ -743,6 +800,11 @@ function write(parent::MatlabHDF5File, name::String, thing) m_write(parent, parent.plain, name, thing) end +function write_subsys(mfile::MatlabHDF5File, subsys_data::Dict{String,Any}) + name = "#subsystem#" + m_write(mfile, mfile.plain, name, subsys_data) +end + ## Type conversion operations ## struct MatlabString end diff --git a/src/MAT_subsys.jl b/src/MAT_subsys.jl index d9396b1..4111bc0 100644 --- a/src/MAT_subsys.jl +++ b/src/MAT_subsys.jl @@ -27,113 +27,102 @@ module MAT_subsys -import ..MAT_types: MatlabStructArray, MatlabOpaque, convert_opaque +import ..MAT_types: MatlabStructArray, MatlabOpaque, convert_opaque, EmptyStruct export Subsystem const FWRAP_VERSION = 4 const MCOS_IDENTIFIER = 0xdd000000 +const matlab_saveobj_ret_types = String[ + "string", + "timetable" +] + mutable struct Subsystem - object_cache::Dict{UInt32,MatlabOpaque} + load_object_cache::Dict{UInt32,MatlabOpaque} + save_object_cache::IdDict{MatlabOpaque,UInt32} num_names::UInt32 # number of mcos_names mcos_names::Vector{String} # Class and Property Names class_id_metadata::Vector{UInt32} - object_id_metadata::Vector{UInt32} - saveobj_prop_metadata::Vector{UInt32} - obj_prop_metadata::Vector{UInt32} + object_id_metadata::Vector{Vector{UInt32}} + saveobj_prop_metadata::Vector{Vector{UInt32}} + obj_prop_metadata::Vector{Vector{UInt32}} dynprop_metadata::Vector{UInt32} _u6_metadata::Vector{UInt32} _u7_metadata::Vector{UInt32} prop_vals_saved::Vector{Any} _c3::Any - _c2::Any + mcos_class_alias_metadata::Any prop_vals_defaults::Any handle_data::Any java_data::Any table_type::Type # Julia type to convert Matlab tables into + # Counters for saving + saveobj_counter::UInt32 + normalobj_counter::UInt32 + obj_id_counter::UInt32 + class_id_counter::UInt32 + function Subsystem() return new( Dict{UInt32,MatlabOpaque}(), + IdDict{MatlabOpaque,UInt32}(), UInt32(0), String[], UInt32[], - UInt32[], - UInt32[], - UInt32[], + Vector{Vector{UInt32}}(), + Vector{Vector{UInt32}}(), + Vector{Vector{UInt32}}(), UInt32[], UInt32[], UInt32[], Any[], nothing, - nothing, + Int32[], nothing, nothing, nothing, Nothing, + UInt32(0), + UInt32(0), + UInt32(0), + UInt32(0), ) end end -function get_object!(subsys::Subsystem, oid::UInt32, classname::String) - if haskey(subsys.object_cache, oid) - # object is already cached, just retrieve it - obj = subsys.object_cache[oid] - else # it's a new object - prop_dict = Dict{String,Any}() - obj = MatlabOpaque(prop_dict, classname) - # cache the new object - subsys.object_cache[oid] = obj - # caching must be done before a next call to `get_properties` to avoid any infinite recursion - merge!(prop_dict, get_properties(subsys, oid)) - end - return obj +function swapped_reinterpret(T::Type, A::AbstractArray{UInt8}, swap_bytes::Bool) + return reinterpret(T, swap_bytes ? reverse(A) : A) end - -function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) - subsys = Subsystem() - return load_subsys!(subsys, subsystem_data, swap_bytes) +# integers are written as uint8 (with swap), interpret as uint32 +function swapped_reinterpret(A::AbstractArray{UInt8}, swap_bytes::Bool) + return swapped_reinterpret(UInt32, A, swap_bytes) end -function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_bytes::Bool) - subsys.handle_data = get(subsystem_data, "handle", nothing) - subsys.java_data = get(subsystem_data, "java", nothing) - mcos_data = get(subsystem_data, "MCOS", nothing) - if mcos_data === nothing - return nothing - end +function init_save!(subsys::Subsystem) + append!(subsys.class_id_metadata, UInt32[0, 0, 0, 0]) + append!(subsys.dynprop_metadata, UInt32[0, 0]) + append!(subsys.mcos_class_alias_metadata, Int32[0]) - if mcos_data isa Tuple - # Backward compatibility with MAT_v5 - mcos_data = mcos_data[2] - end - fwrap_metadata::Vector{UInt8} = vec(mcos_data[1, 1]) - - version = swapped_reinterpret(fwrap_metadata[1:4], swap_bytes)[1] - if version <= 1 || version > FWRAP_VERSION - error("Cannot read subsystem: Unsupported FileWrapper version: $version") - end + # These are Vector{Vector{uint32}} + # need mutable inner vectors to handle nested properties + push!(subsys.object_id_metadata, UInt32[0, 0, 0, 0, 0, 0]) + push!(subsys.saveobj_prop_metadata, UInt32[0, 0]) + push!(subsys.obj_prop_metadata, UInt32[0, 0]) - subsys.num_names = swapped_reinterpret(fwrap_metadata[5:8], swap_bytes)[1] - load_mcos_names!(subsys, fwrap_metadata) - - load_mcos_regions!(subsys, fwrap_metadata, swap_bytes) + subsys._c3 = Any[] + subsys.prop_vals_defaults = Any[] - if version == 2 - subsys.prop_vals_saved = mcos_data[3:(end - 1), 1] - elseif version == 3 - subsys.prop_vals_saved = mcos_data[3:(end - 2), 1] - subsys._c2 = mcos_data[end - 1, 1] - else - subsys.prop_vals_saved = mcos_data[3:(end - 3), 1] - subsys._c3 = mcos_data[end - 2, 1] - end - - subsys.prop_vals_defaults = mcos_data[end, 1] return subsys end +function load_subsys!(subsystem_data::Dict{String,Any}, swap_bytes::Bool) + subsys = Subsystem() + return load_subsys!(subsys, subsystem_data, swap_bytes) +end + # Class and Property Names are stored as list of null-terminated strings function load_mcos_names!(subsys::Subsystem, fwrap_metadata::AbstractArray{UInt8}) start = 41 @@ -160,15 +149,15 @@ function load_mcos_regions!( subsys.class_id_metadata = swapped_reinterpret( get_region(fwrap_metadata, region_offsets, 1), swap_bytes ) - subsys.saveobj_prop_metadata = swapped_reinterpret( + subsys.saveobj_prop_metadata = [swapped_reinterpret( get_region(fwrap_metadata, region_offsets, 2), swap_bytes - ) - subsys.object_id_metadata = swapped_reinterpret( + )] + subsys.object_id_metadata = [swapped_reinterpret( get_region(fwrap_metadata, region_offsets, 3), swap_bytes - ) - subsys.obj_prop_metadata = swapped_reinterpret( + )] + subsys.obj_prop_metadata = [swapped_reinterpret( get_region(fwrap_metadata, region_offsets, 4), swap_bytes - ) + )] subsys.dynprop_metadata = swapped_reinterpret( get_region(fwrap_metadata, region_offsets, 5), swap_bytes ) @@ -192,12 +181,42 @@ function get_region( return fwrap_metadata[(region_offsets[region] + 1):region_offsets[region + 1]] end -function swapped_reinterpret(T::Type, A::AbstractArray{UInt8}, swap_bytes::Bool) - return reinterpret(T, swap_bytes ? reverse(A) : A) -end -# integers are written as uint8 (with swap), interpret as uint32 -function swapped_reinterpret(A::AbstractArray{UInt8}, swap_bytes::Bool) - return swapped_reinterpret(UInt32, A, swap_bytes) +function load_subsys!(subsys::Subsystem, subsystem_data::Dict{String,Any}, swap_bytes::Bool) + subsys.handle_data = get(subsystem_data, "handle", nothing) + subsys.java_data = get(subsystem_data, "java", nothing) + mcos_data = get(subsystem_data, "MCOS", nothing) + if mcos_data === nothing + return nothing + end + + if mcos_data isa Tuple + # Backward compatibility with MAT_v5 + mcos_data = mcos_data[2] + end + fwrap_metadata::Vector{UInt8} = vec(mcos_data[1, 1]) + + version = swapped_reinterpret(fwrap_metadata[1:4], swap_bytes)[1] + if version <= 1 || version > FWRAP_VERSION + error("Cannot read subsystem: Unsupported FileWrapper version: $version") + end + + subsys.num_names = swapped_reinterpret(fwrap_metadata[5:8], swap_bytes)[1] + load_mcos_names!(subsys, fwrap_metadata) + + load_mcos_regions!(subsys, fwrap_metadata, swap_bytes) + + if version == 2 + subsys.prop_vals_saved = mcos_data[3:(end - 1), 1] + elseif version == 3 + subsys.prop_vals_saved = mcos_data[3:(end - 2), 1] + subsys.mcos_class_alias_metadata = mcos_data[end - 1, 1] + else + subsys.prop_vals_saved = mcos_data[3:(end - 3), 1] + subsys._c3 = mcos_data[end - 2, 1] + end + + subsys.prop_vals_defaults = mcos_data[end, 1] + return subsys end function get_classname(subsys::Subsystem, class_id::UInt32) @@ -215,31 +234,7 @@ function get_classname(subsys::Subsystem, class_id::UInt32) end function get_object_metadata(subsys::Subsystem, object_id::UInt32) - return subsys.object_id_metadata[(object_id * 6 + 1):(object_id * 6 + 6)] -end - -function get_default_properties(subsys::Subsystem, class_id::UInt32) - default_props = Dict{String,Any}(subsys.prop_vals_defaults[class_id + 1, 1]) - for (key, value) in default_props - default_props[key] = update_nested_props!(value, subsys) - end - return default_props -end - -function get_property_idxs(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) - prop_field_idxs = - saveobj_ret_type ? subsys.saveobj_prop_metadata : subsys.obj_prop_metadata - nfields = 3 - offset = 1 - while obj_type_id > 0 - nprops = prop_field_idxs[offset] - offset += 1 + (nfields * nprops) - offset += (offset + 1) % 2 # Padding - obj_type_id -= 1 - end - nprops = prop_field_idxs[offset] - offset += 1 - return prop_field_idxs[offset:(offset + nprops * nfields - 1)] + return subsys.object_id_metadata[1][(object_id * 6 + 1):(object_id * 6 + 6)] end update_nested_props!(prop_value, subsys::Subsystem) = prop_value @@ -276,6 +271,30 @@ function update_nested_props!(prop_value::Array{UInt32}, subsys::Subsystem) end end +function get_default_properties(subsys::Subsystem, class_id::UInt32) + default_props = Dict{String,Any}(subsys.prop_vals_defaults[class_id + 1, 1]) + for (key, value) in default_props + default_props[key] = update_nested_props!(value, subsys) + end + return default_props +end + +function get_property_idxs(subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool) + prop_field_idxs = + saveobj_ret_type ? subsys.saveobj_prop_metadata[1] : subsys.obj_prop_metadata[1] + nfields = 3 + offset = 1 + while obj_type_id > 0 + nprops = prop_field_idxs[offset] + offset += 1 + (nfields * nprops) + offset += (offset + 1) % 2 # Padding + obj_type_id -= 1 + end + nprops = prop_field_idxs[offset] + offset += 1 + return prop_field_idxs[offset:(offset + nprops * nfields - 1)] +end + function get_saved_properties( subsys::Subsystem, obj_type_id::UInt32, saveobj_ret_type::Bool ) @@ -349,6 +368,21 @@ function get_properties(subsys::Subsystem, object_id::UInt32) return prop_map end +function get_object!(subsys::Subsystem, oid::UInt32, classname::String) + if haskey(subsys.load_object_cache, oid) + # object is already cached, just retrieve it + obj = subsys.load_object_cache[oid] + else # it's a new object + prop_dict = Dict{String,Any}() + obj = MatlabOpaque(prop_dict, classname) + # cache the new object + subsys.load_object_cache[oid] = obj + # caching must be done before a next call to `get_properties` to avoid any infinite recursion + merge!(prop_dict, get_properties(subsys, oid)) + end + return obj +end + function load_mcos_object(metadata::Any, type_name::String, subsys::Subsystem) @warn "Expected MCOS metadata to be an Array{UInt32}, got $(typeof(metadata)). Returning metadata." return metadata @@ -394,4 +428,333 @@ function load_mcos_object(metadata::Array{UInt32}, type_name::String, subsys::Su end end -end \ No newline at end of file +function set_mcos_name!(subsys::Subsystem, name::String) + idx = findfirst(isequal(name), subsys.mcos_names) + if idx !== nothing + return UInt32(idx) + else + push!(subsys.mcos_names, name) + subsys.num_names += UInt32(1) + return subsys.num_names + end +end + +function check_valid_property_name(s::AbstractString) + if match(r"^[a-zA-Z][a-zA-Z0-9_]*$", s) === nothing + error("Invalid property name \"$s\": property names must start with a letter and contain only alphanumeric characters and underscore") + end +end + +function set_class_id!(subsys::Subsystem, classname::String) + # number of existing class entries (each class has 4 UInt32 metadata entries) + class_count = UInt32(length(subsys.class_id_metadata) รท 4 - 1) # skip class_id = 0 case + + # Check if class name already exists + for cid in UInt32(1):UInt32(class_count) + existing_name = get_classname(subsys, cid) + if existing_name == classname + return cid + end + end + + # Add new class metadata + subsys.class_id_counter += UInt32(1) + # split namespace and class name at last dot + pos = findlast(==('.'), classname) + if pos === nothing + namespace = "" + cname = classname + else + namespace = classname[1:pos-1] + cname = classname[pos+1:end] + end + + cname_idx = set_mcos_name!(subsys, cname) + namespace_idx = pos === nothing ? UInt32(0) : set_mcos_name!(subsys, namespace) + + append!(subsys.class_id_metadata, UInt32[namespace_idx, cname_idx, 0, 0]) + append!(subsys.mcos_class_alias_metadata, Int32[0]) # Placeholder for no aliases + + return subsys.class_id_counter +end + +save_nested_props(prop_value, subsys::Subsystem) = prop_value + +function save_nested_props( + prop_value::Union{AbstractDict,MatlabStructArray}, subsys::Subsystem +) + # Save nested objects in structs + for (key, value) in prop_value + prop_value[key] = save_nested_props(value, subsys) + end + return prop_value +end + +function save_nested_props(prop_value::Array{Any}, subsys::Subsystem) + # Save nested objects in a Cell + for i in eachindex(prop_value) + prop_value[i] = save_nested_props(prop_value[i], subsys) + end + return prop_value +end + +function save_nested_props(prop_value::Union{MatlabOpaque, Array{MatlabOpaque}}, subsys::Subsystem) + # Nested objects are saved by the uint32 Matrix signature + # however they don't have any mxOPAQUE_CLASS headers + # so we search within containers here + + # FIXME: Does this overwrite prop_value from the user dict? + # Might have to create a copy instead - Test needed + prop_value = set_mcos_object_metadata(subsys, prop_value) + return prop_value +end + +function serialize_object_props!(subsys::Subsystem, obj::MatlabOpaque, obj_prop_metadata::Vector{UInt32}) + + object_id = subsys.obj_id_counter + nprops = length(obj) + + # property metadata format: [num_props, (prop_name_idx, prop_type, prop_value_idx)*] + push!(obj_prop_metadata, UInt32(nprops)) + + for (prop_name, prop_value) in obj + if startswith(prop_name, "__dynamic_property_") + @warn "Dynamic properties are not supported when writing: skipping $prop_name" + continue + end + + check_valid_property_name(prop_name) + + field_name_idx = set_mcos_name!(subsys, prop_name) + prop_value = save_nested_props(prop_value, subsys) + + prop_vals = UInt32[field_name_idx, 1, 0] + + cell_idx = length(subsys.prop_vals_saved) # these are zero-indexed in matlab + push!(subsys.prop_vals_saved, prop_value) + prop_vals[3] = cell_idx + append!(obj_prop_metadata, prop_vals) + end + + if length(obj_prop_metadata) % 2 == 1 + push!(obj_prop_metadata, UInt32(0)) # padding + end + + # placeholder for dynamic props + append!(subsys.dynprop_metadata, UInt32[0, 0]) + + ndeps = subsys.obj_id_counter - object_id + return ndeps +end + +function set_object_id(subsys::Subsystem, obj::MatlabOpaque, saveobj_ret_type=false) + + if length(obj) == 0 + # This is a deleted object + # MATLAB keeps weak references to deleted objects for some reason + class_id = set_class_id!(subsys, obj.class) + obj_id = 0 + return obj_id, class_id + end + + if haskey(subsys.save_object_cache, obj) + class_id = set_class_id!(subsys, obj.class) + obj_id = subsys.save_object_cache[obj] + return obj_id, class_id + end + + subsys.obj_id_counter += UInt32(1) + mat_obj_id = subsys.obj_id_counter + subsys.save_object_cache[obj] = mat_obj_id + + prop_metadata = UInt32[] + saveobj_id = 0 + normobj_id = 0 + + if saveobj_ret_type + subsys.saveobj_counter += UInt32(1) + saveobj_id = subsys.saveobj_counter + push!(subsys.saveobj_prop_metadata, prop_metadata) + else + subsys.normalobj_counter += UInt32(1) + normobj_id = subsys.normalobj_counter + push!(subsys.obj_prop_metadata, prop_metadata) + end + + obj_id_metadata = zeros(UInt32, 6) + push!(subsys.object_id_metadata, obj_id_metadata) + + if saveobj_ret_type && !(length(obj) == 1 && haskey(obj, "any")) + error("Object of class $(obj.classname) marked with a saveobj return type must have a single property 'any' containing the return value of its saveobj method") + end + + ndeps = serialize_object_props!(subsys, obj, prop_metadata) + + class_id = set_class_id!(subsys, obj.class) + obj_id_metadata[1] = class_id + if saveobj_ret_type + obj_id_metadata[4] = saveobj_id + else + obj_id_metadata[5] = normobj_id + end + + # it works idk how + obj_id_metadata[6] = mat_obj_id + ndeps + for i in 1:ndeps + obj_id = subsys.obj_id_counter - ndeps + i + subsys.object_id_metadata[obj_id + 1][6] -= 1 + end + + return mat_obj_id, class_id +end + +function create_mcos_metadata_array(dims::Tuple{Vararg{Int}}, arr_ids::Vector{UInt32}, class_id::UInt32) + ndims = length(dims) + metadata = UInt32[] + push!(metadata, MCOS_IDENTIFIER) + push!(metadata, UInt32(ndims)) + for d in dims + push!(metadata, UInt32(d)) + end + for oid in arr_ids + push!(metadata, oid) + end + push!(metadata, class_id) + + return reshape(metadata, :, 1) +end + +function set_mcos_object_metadata(subsys::Subsystem, obj::Array{MatlabOpaque}) + + arr_ids = UInt32[] + if length(obj) == 0 + # TODO: Handle 1x0, 0x0, 0x1 objects + # placeholder for empty array case + dims = size(obj) + return create_mcos_metadata_array(dims, UInt32(0), UInt32(0)) + end + classname = first(obj).class + + # this is not needed but added for consistency + # future update can support mentioning classes with saveobj methods + saveobj_ret_type = classname in matlab_saveobj_ret_types + + class_id = UInt32(0) + for obj_elem in obj + obj_id, class_id = set_object_id(subsys, obj_elem, saveobj_ret_type) + append!(arr_ids, obj_id) + end + + dims = size(obj) + return create_mcos_metadata_array(dims, arr_ids, class_id) +end + +function set_mcos_object_metadata(subsys::Subsystem, obj::MatlabOpaque) + + classname = obj.class + saveobj_ret_type = classname in matlab_saveobj_ret_types + + object_id, class_id = set_object_id(subsys, obj, saveobj_ret_type) + dims = (1, 1) + return create_mcos_metadata_array(dims, [object_id], class_id) +end + +function set_fwrap_metadata!(subsys::Subsystem) + + version_bytes = vec(reinterpret(UInt8, UInt32[FWRAP_VERSION])) + num_names_bytes = copy(vec(reinterpret(UInt8, UInt32[subsys.num_names]))) + + region_offsets = zeros(UInt32, 8) + + names_str = isempty(subsys.mcos_names) ? "" : join(subsys.mcos_names, '\0') * '\0' + names_bytes = collect(codeunits(names_str)) + pad_len = (8 - (length(names_bytes) % 8)) % 8 + if pad_len > 0 + append!(names_bytes, zeros(UInt8, pad_len)) + end + + region1_bytes = vec(reinterpret(UInt8, subsys.class_id_metadata)) + region_offsets[1] = UInt32(40 + length(names_bytes)) + region_offsets[2] = region_offsets[1] + UInt32(length(region1_bytes)) + + # This region (including id=0) does not exist if there are no saveobj type objects + region2_bytes = UInt8[] + if length(subsys.saveobj_prop_metadata) > 1 + for sub in subsys.saveobj_prop_metadata + append!(region2_bytes, vec(reinterpret(UInt8, sub))) + end + end + region_offsets[3] = region_offsets[2] + UInt32(length(region2_bytes)) + + region3_bytes = UInt8[] + for sub in subsys.object_id_metadata + append!(region3_bytes, vec(reinterpret(UInt8, sub))) + end + region_offsets[4] = region_offsets[3] + UInt32(length(region3_bytes)) + + region4_bytes = UInt8[] + for sub in subsys.obj_prop_metadata + append!(region4_bytes, vec(reinterpret(UInt8, sub))) + end + region_offsets[5] = region_offsets[4] + UInt32(length(region4_bytes)) + + region5_bytes = vec(reinterpret(UInt8, subsys.dynprop_metadata)) + region_offsets[6] = region_offsets[5] + UInt32(length(region5_bytes)) + + region6_bytes = UInt8[] + region_offsets[7] = region_offsets[6] + UInt32(length(region6_bytes)) + + region7_bytes = zeros(UInt8, 8) + region_offsets[8] = region_offsets[7] + UInt32(length(region7_bytes)) + + fwrap = Vector{UInt8}() + append!(fwrap, version_bytes) + append!(fwrap, num_names_bytes) + append!(fwrap, vec(reinterpret(UInt8, region_offsets))) + append!(fwrap, names_bytes) + append!(fwrap, region1_bytes) + append!(fwrap, region2_bytes) + append!(fwrap, region3_bytes) + append!(fwrap, region4_bytes) + append!(fwrap, region5_bytes) + append!(fwrap, region6_bytes) + append!(fwrap, region7_bytes) + + return fwrap +end + +function set_fwrap_data!(subsys::Subsystem) + + fwrap_data = Any[] + fwrap_metadata = set_fwrap_metadata!(subsys) + push!(fwrap_data, reshape(fwrap_metadata, :, 1)) + push!(fwrap_data, reshape(Any[], 0, 0)) + append!(fwrap_data, subsys.prop_vals_saved) + + empty_struct = EmptyStruct([1, 0]) + for i in 0:subsys.class_id_counter + push!(subsys._c3, empty_struct) + push!(subsys.prop_vals_defaults, empty_struct) + end + + push!(fwrap_data, reshape(subsys._c3, :, 1)) + push!(fwrap_data, reshape(subsys.mcos_class_alias_metadata, :, 1)) + push!(fwrap_data, reshape(subsys.prop_vals_defaults, :, 1)) + + fw_obj = MatlabOpaque(Dict("__filewrapper__" => reshape(fwrap_data, :, 1)), "FileWrapper__") + return fw_obj +end + +function set_subsystem_data!(subsys::Subsystem) + if subsys.class_id_counter == 0 + # No MCOS objects to serialize + return nothing + end + + fwrap = set_fwrap_data!(subsys) + subsys_struct = Dict{String,Any}() + subsys_struct["MCOS"] = fwrap + return subsys_struct +end + +end diff --git a/src/MAT_types.jl b/src/MAT_types.jl index a7c0674..3c41c81 100644 --- a/src/MAT_types.jl +++ b/src/MAT_types.jl @@ -50,8 +50,8 @@ export MatlabTable Data structure to store matlab struct arrays, which stores the field names separate from the field values. The field values are stored as columns of `Array{Any,N}` per Matlab field, which is how MAT files store these structures. -These are distinct from cell arrays of structs, -which are handled as in MAT.jl as `Array{Any,N}` with `Dict{String,Any}` inside, +These are distinct from cell arrays of structs, +which are handled as in MAT.jl as `Array{Any,N}` with `Dict{String,Any}` inside, for example `Any[Dict("x"=>1), Dict("x"=>2)]`. Old class object arrays can be handled by providing a non-empty class name. @@ -261,6 +261,14 @@ struct StructArrayField{N} end dimension(::StructArrayField{N}) where {N} = N +""" +Internal Marker for Empty Structs with dimensions like 1x0 or 0x0 +""" +struct EmptyStruct + dims::Vector{UInt64} +end + + """ MatlabClassObject( d::Dict{String, Any}, diff --git a/test/write.jl b/test/write.jl index 21f32b3..8910329 100644 --- a/test/write.jl +++ b/test/write.jl @@ -1,4 +1,4 @@ -using MAT +using MAT, Test tmpfile = string(tempname(), ".mat") @@ -145,7 +145,7 @@ test_write(sd) # which are not compressible by themselves! test_compression_effective(Dict("data" => fill(1.0, 1000))) -# test adjoint/reshape array +# test adjoint/reshape array test_write(Dict("adjoint_arr"=>[1 2 3;4 5 6;7 8 9]')) test_write(Dict("reshape_arr"=>reshape([1 2 3;4 5 6;7 8 9]',1,9))) @@ -193,4 +193,111 @@ nt_read = matread(tmpfile)["nt"] matwrite(tmpfile, Dict("class_array" => carr)) carr_read = matread(tmpfile)["class_array"] @test carr_read == MatlabStructArray(carr) -end \ No newline at end of file +end + +@testset "MatlabOpaque simple" begin + d = Dict{String,Any}("a" => 1, "b" => Any[1.0, 2.0]) + obj = MatlabOpaque(d, "TestClass") + var_dict = Dict("var" => obj) + + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, var_dict) + read_var = matread(tmpfile) + + @test haskey(read_var, "var") + @test isa(read_var["var"], MatlabOpaque) + @test read_var["var"].class == obj.class + + @test haskey(read_var["var"], "a") + @test haskey(read_var["var"], "b") + @test read_var["var"]["a"] == obj["a"] + @test isequal(read_var["var"]["b"], obj["b"]) + end +end + +@testset "Empty Struct 1x1" begin + var_dict = Dict("empty_struct" => Dict{String,Any}()) + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, var_dict) + read_var = matread(tmpfile) + + @test haskey(read_var, "empty_struct") + @test isa(read_var["empty_struct"], Dict{String,Any}) + @test length(keys(read_var["empty_struct"])) == 0 + end +end + +@testset "MatlabOpaque handle" begin + d = Dict{String,Any}("a" => 1, "b" => Any[1.0, 2.0]) + obj = MatlabOpaque(d, "TestClassHandle") + var_dict = Dict("var1" => obj, "var2" => obj) + + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, var_dict) + read_var = matread(tmpfile) + + @test haskey(read_var, "var1") + @test haskey(read_var, "var2") + @test isa(read_var["var1"], MatlabOpaque) + @test isa(read_var["var2"], MatlabOpaque) + @test read_var["var1"] === read_var["var2"] # same object + end +end + +@testset "MatlabOpaque Array" begin + d1 = Dict{String, Any}("a" => 1) + d2 = Dict{String, Any}("a" => 2) + obj1 = MatlabOpaque(d1, "TestClassArray") + obj2 = MatlabOpaque(d2, "TestClassArray") + obj_arr = Array{MatlabOpaque}(undef, 2, 1) + obj_arr[1, 1] = obj1 + obj_arr[2, 1] = obj2 + var_dict = Dict("obj_array" => obj_arr) + + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, var_dict) + read_var = matread(tmpfile) + + @test haskey(read_var, "obj_array") + @test isa(read_var["obj_array"], Array{MatlabOpaque}) + @test size(read_var["obj_array"]) == (2, 1) + @test read_var["obj_array"][1, 1]["a"] == 1 + @test read_var["obj_array"][2, 1]["a"] == 2 + end +end + +@testset "MatlabOpaque Nested object" begin + inner_dict = Dict{String,Any}("a" => 1, "b" => 2) + inner_obj = MatlabOpaque(inner_dict, "InnerClass") + outer_dict = Dict{String,Any}("inner" => inner_obj, "c" => 3) + outer_obj = MatlabOpaque(outer_dict, "OuterClass") + var_dict = Dict("outer_obj" => outer_obj) + + mktempdir() do tmpdir + tmpfile = joinpath(tmpdir, "test.mat") + matwrite(tmpfile, var_dict) + read_var = matread(tmpfile) + + @test haskey(read_var, "outer_obj") + @test isa(read_var["outer_obj"], MatlabOpaque) + @test read_var["outer_obj"].class == outer_obj.class + + @test haskey(read_var["outer_obj"], "inner") + @test haskey(read_var["outer_obj"], "c") + @test read_var["outer_obj"]["c"] == outer_obj["c"] + + inner_read = read_var["outer_obj"]["inner"] + @test isa(inner_read, MatlabOpaque) + @test inner_read.class == inner_obj.class + + @test haskey(inner_read, "a") + @test haskey(inner_read, "b") + @test inner_read["a"] == inner_obj["a"] + @test inner_read["b"] == inner_obj["b"] + end + +end