Skip to content

Writing MatlabOpaque objects to MAT-files #208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matthijscox merged 10 commits into from
Dec 2, 2025
Merged

Writing MatlabOpaque objects to MAT-files #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
dc7e37f
Rearrange methods for readability
foreverallama Nov 21, 2025
4026a63
Add some subsys methods to save objects
foreverallama Nov 24, 2025
dd1c995
Full subsystem code, some updates to writers to support writing subsy…
foreverallama Nov 25, 2025
32c678a
Some fixes to writing filewrapper metadata
foreverallama Nov 26, 2025
8416fc2
Fix dimensions of data in filewrapper
foreverallama Nov 27, 2025
7b846dc
Write nested objects in subsystem
foreverallama Nov 27, 2025
3c85b70
Improve MAT_subsys readability
foreverallama Nov 28, 2025
46c56aa
Fix: Write empty dicts as 0x0 structs in HDF5
foreverallama Nov 28, 2025
1d9e8bb
Update empty struct handling; Add tests
foreverallama Nov 29, 2025
74ca2bc
Add test for empty struct 1x1
foreverallama Nov 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 32 additions & 20 deletions src/MAT.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,31 +204,43 @@ Write a dictionary containing variable names as keys and values as values
to a Matlab file, opening and closing it automatically.
"""
function matwrite(filename::AbstractString, dict::AbstractDict{S, T}; compress::Bool = false, version::String ="v7.3") where {S, T}
if version == "v4"
file = open(filename, "w")
m = MAT_v4.Matlabv4File(file, false)
_write_dict(m, dict)
elseif version == "v7.3"
file = matopen(filename, "w"; compress = compress)
_write_dict(file, dict)
else
error("writing for \"$(version)\" is not supported")
file = nothing
try
if version == "v4"
file = open(filename, "w")
file = MAT_v4.Matlabv4File(file, false)
_write_dict(file, dict)
elseif version == "v7.3"
file = matopen(filename, "w"; compress = compress)
_write_dict(file, dict)
else
error("writing for \"$(version)\" is not supported")
end
finally
if file !== nothing
close(file)
end
end
end

function _write_dict(fileio, dict::AbstractDict)
try
for (k, v) in dict
local kstring
try
kstring = ascii(convert(String, k))
catch x
error("matwrite requires a Dict with ASCII keys")
end
write(fileio, kstring, v)

for (k, v) in dict
local kstring
try
kstring = ascii(convert(String, k))
catch x
error("matwrite requires a Dict with ASCII keys")
end
write(fileio, kstring, v)
end

if hasproperty(fileio, :subsystem) && fileio.subsystem !== nothing
# will always be nothing for MATv4 so we can ignore that case
subsys_data = MAT_subsys.set_subsystem_data!(fileio.subsystem)
if subsys_data !== nothing
MAT_HDF5.write_subsys(fileio, subsys_data)
end
finally
close(fileio)
end
end

Expand Down
76 changes: 69 additions & 7 deletions src/MAT_HDF5.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import HDF5: Reference
import Dates
import Tables
import PooledArrays: PooledArray
import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable
import ..MAT_types: MatlabStructArray, StructArrayField, convert_struct_array, MatlabClassObject, MatlabOpaque, MatlabTable, EmptyStruct

const HDF5Parent = Union{HDF5.File, HDF5.Group}
const HDF5BitsOrBool = Union{HDF5.BitsType,Bool}
Expand Down Expand Up @@ -130,10 +130,12 @@ function matopen(filename::AbstractString, rd::Bool, wr::Bool, cr::Bool, tr::Boo
close(g)
end
subsys_refs = "#subsystem#"
if haskey(fid.plain, subsys_refs)
if rd && haskey(fid.plain, subsys_refs)
fid.subsystem.table_type = table
subsys_data = m_read(fid.plain[subsys_refs], fid.subsystem)
MAT_subsys.load_subsys!(fid.subsystem, subsys_data, endian_indicator)
elseif wr
MAT_subsys.init_save!(fid.subsystem)
end
fid
end
Expand Down Expand Up @@ -464,7 +466,7 @@ function _normalize_arr(x)
end

# Write a scalar or array
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}) where {T<:HDF5BitsOrBool}
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::Union{T, Complex{T}, AbstractArray{T}, AbstractArray{Complex{T}}}, ) where {T<:HDF5BitsOrBool}
data = _normalize_arr(data)
if isempty(data)
m_writeempty(parent, name, data)
Expand Down Expand Up @@ -541,14 +543,19 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, c::Abs
end

# Write cell arrays
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}) where T
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, data::AbstractArray{T}, object_decode::UInt32=UInt32(0)) where T
data = _normalize_arr(data)
refs = _write_references!(mfile, parent, data)
# Write the references as the chosen variable
cset, ctype = create_dataset(parent, name, refs)
try
write_dataset(cset, ctype, refs)
write_attribute(cset, name_type_attr_matlab, "cell")
if object_decode == UInt32(3)
write_attribute(cset, object_decode_attr_matlab, object_decode)
Copy link
Member

@matthijscox matthijscox Nov 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this (FileWrapper) case tested?

Copy link
Member

@matthijscox matthijscox Nov 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can this code even be reached by users?

Copy link
Contributor Author

@foreverallama foreverallama Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Part 1 addressed in the other comment. As to your second question, its not reachable by users. The FileWrapper object is some kind of special container used by MATLAB which stores data in a Cell array. This is just a simple workaround to write a cell array for MatlabOpaque types whose data is a cell array.

As far as I know, no other MatlabOpaque types use cell arrays to store (meta)data, so it will only execute for FileWrapper.

write_attribute(cset, name_type_attr_matlab, "FileWrapper__")
else
write_attribute(cset, name_type_attr_matlab, "cell")
end
finally
close(ctype)
close(cset)
Expand Down Expand Up @@ -680,8 +687,34 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::M
end
end

# Write empty (zero-dimensional) structs with no fields
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, s::EmptyStruct)
dset, dtype = create_dataset(parent, name, s.dims)
try
write_attribute(dset, empty_attr_matlab, 0x01)
write_attribute(dset, name_type_attr_matlab, "struct")
write_dataset(dset, dtype, s.dims)
finally
close(dtype); close(dset)
end
end

# Write a struct from arrays of keys and values
function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, k::Vector{String}, v::Vector)
if length(k) == 0
Copy link
Contributor Author

@foreverallama foreverallama Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the ability to write empty dicts as 0x0 structs over here. The subsystem data works with 0x0 structs as well in MATLAB.

This eliminates the need for any new struct markers for now.

Copy link
Member

@matthijscox matthijscox Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alright. Are 0x0 structs also read as empty dicts?

Copy link
Contributor Author

@foreverallama foreverallama Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup, all of 1x0, 0x1 and 0x0 are read as empty dicts already. But I hardcoded 0x0 for empty dict during write.

Copy link
Contributor Author

@foreverallama foreverallama Nov 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So this might go against the point raised in #214 where an empty dict is being written as a 1x1 struct with no fields. Instead I guess we could either:

  • Write empty dicts as 1x1 structs with no fields. For 1x0, 0x0 dims we could maybe use a special key __dims__ that contains a tuple with the dimensions. I think its very unlikely to require reading 1x0, 0x0 structs (without fields), we just need write functionality
  • Create a new marker for empty structs with no fields
  • Update MatlabStructArray to handle empty structs

What do you suggest?

Copy link
Member

@matthijscox matthijscox Nov 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a rather special corner case. I'm okay with a new type that we keep internal for the writing, and then we can always consider to refactor later:

struct EmptyStruct
   dims::Vector{Int}
end

# empty struct
adata = UInt64[1, 1]
dset, dtype = create_dataset(parent, name, adata)
try
write_attribute(dset, empty_attr_matlab, 0x01)
write_attribute(dset, name_type_attr_matlab, "struct")
write_dataset(dset, dtype, adata)
finally
close(dtype); close(dset)
end
return
end

g = create_group(parent, name)
try
write_attribute(g, name_type_attr_matlab, "struct")
Expand Down Expand Up @@ -719,11 +752,35 @@ function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, dat::D
end

function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::MatlabOpaque)
error("writing of MatlabOpaque types is not yet supported")
if obj.class == "FileWrapper__"
m_write(mfile, parent, name, obj["__filewrapper__"], UInt32(3))
Copy link
Member

@matthijscox matthijscox Nov 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this FileWrapper case tested?

Copy link
Contributor Author

@foreverallama foreverallama Dec 1, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's implicitly tested with the object tests. All object data + metadata is stored within this FileWrapper__ object in a MAT-file, so it is not possible to load an object without having the FileWrapper object written correctly to a MAT-file.

return
end

metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj)
dset, dtype = create_dataset(parent, name, metadata)
try
write_dataset(dset, dtype, metadata)
write_attribute(dset, name_type_attr_matlab, obj.class)
write_attribute(dset, object_type_attr_matlab, UInt32(3))
finally
close(dset)
close(dtype)
end
end

function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, obj::AbstractArray{MatlabOpaque})
error("writing of MatlabOpaque types is not yet supported")
metadata = MAT_subsys.set_mcos_object_metadata(mfile.subsystem, obj)
dset, dtype = create_dataset(parent, name, metadata)
try
# TODO: Handle empty array case
write_dataset(dset, dtype, metadata)
write_attribute(dset, name_type_attr_matlab, first(obj).class)
write_attribute(dset, object_type_attr_matlab, UInt32(3))
finally
close(dset)
close(dtype)
end
end

function m_write(mfile::MatlabHDF5File, parent::HDF5Parent, name::String, arr::PooledArray)
Expand All @@ -743,6 +800,11 @@ function write(parent::MatlabHDF5File, name::String, thing)
m_write(parent, parent.plain, name, thing)
end

function write_subsys(mfile::MatlabHDF5File, subsys_data::Dict{String,Any})
name = "#subsystem#"
m_write(mfile, mfile.plain, name, subsys_data)
end

## Type conversion operations ##

struct MatlabString end
Expand Down
Loading
Loading