diff --git a/res/fs_binary_msgpack1.png b/res/fs_binary_msgpack1.png new file mode 100644 index 0000000..b536b20 Binary files /dev/null and b/res/fs_binary_msgpack1.png differ diff --git a/res/fs_binary_msgpack2.png b/res/fs_binary_msgpack2.png new file mode 100644 index 0000000..cd7a9a5 Binary files /dev/null and b/res/fs_binary_msgpack2.png differ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5adca01..93cf2d6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,3 +23,15 @@ target_link_libraries(solanaceae_message_fragment_store PUBLIC ######################################## +add_executable(convert_message_object_store + ./convert_frag_to_obj.cpp +) + +target_link_libraries(convert_message_object_store PUBLIC + solanaceae_object_store + solanaceae_object_store_backend_filesystem + solanaceae_message_fragment_store +) + +######################################## + diff --git a/src/convert_frag_to_obj.cpp b/src/convert_frag_to_obj.cpp new file mode 100644 index 0000000..54746e5 --- /dev/null +++ b/src/convert_frag_to_obj.cpp @@ -0,0 +1,167 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#include + +int main(int argc, const char** argv) { + if (argc != 3) { + std::cerr << "wrong paramter count, do " << argv[0] << " \n"; + return 1; + } + + if (!std::filesystem::is_directory(argv[1])) { + std::cerr << "input folder is no folder\n"; + } + + std::filesystem::create_directories(argv[2]); + + // we are going to use 2 different OS for convineance, but could be done with 1 too + ObjectStore2 os_src; + ObjectStore2 os_dst; + + backend::FilesystemStorage fsb_src(os_src, argv[1]); + backend::FilesystemStorage fsb_dst(os_dst, argv[2]); + + Contact3Registry cr; // dummy + RegistryMessageModel rmm(cr); // dummy + // they only exist for the serializers (for now) + // TODO: version + MessageSerializerNJ msnj_src{cr, {}, {}}; + MessageFragmentStore mfs_src(cr, rmm, os_src, fsb_src, msnj_src); + MessageSerializerNJ msnj_dst{cr, {}, {}}; + MessageFragmentStore mfs_dst(cr, rmm, os_dst, fsb_dst, msnj_dst); + + // add message fragment store too (adds meta?) + + // hookup events + struct EventListener : public ObjectStoreEventI { + ObjectStore2& _os_src; + backend::FilesystemStorage& _fsb_src; + + ObjectStore2& _os_dst; + backend::FilesystemStorage& _fsb_dst; + + EventListener( + ObjectStore2& os_src, + backend::FilesystemStorage& fsb_src, + ObjectStore2& os_dst, + backend::FilesystemStorage& fsb_dst + ) : + _os_src(os_src), + _fsb_src(fsb_src), + _os_dst(os_dst), + _fsb_dst(fsb_dst) + { + _os_src.subscribe(this, ObjectStore_Event::object_construct); + _os_src.subscribe(this, ObjectStore_Event::object_update); + } + + protected: // os + bool onEvent(const ObjectStore::Events::ObjectConstruct& e) override { + assert(e.e.all_of()); + assert(e.e.all_of()); + + // !! we read the obj first, so we can discard empty objects + // technically we could just copy the file, but meh + // read src and write dst data + std::vector tmp_buffer; + std::function cb = [&tmp_buffer](const ByteSpan buffer) { + tmp_buffer.insert(tmp_buffer.end(), buffer.cbegin(), buffer.cend()); + }; + if (!_fsb_src.read(e.e, cb)) { + std::cerr << "failed to read obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + + if (tmp_buffer.empty()) { + std::cerr << "discarded empty obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + { // try getting lucky and see if its an empty json + const auto j = nlohmann::json::parse(tmp_buffer, nullptr, false); + if (j.is_array() && j.empty()) { + std::cerr << "discarded empty json array obj '" << bin2hex(e.e.get().v) << "'\n"; + return false; + } + } + + // WARNING: manual and hardcoded + // manually upconvert message json to msgpack (v1 to v2) + if (true && e.e.get().v == 1) { + // TODO: error handling + const auto j = nlohmann::json::parse(tmp_buffer); + + if (true) { + e.e.replace(uint16_t(2)); + + // overwrite og + tmp_buffer = nlohmann::json::to_msgpack(j); + } + } + + // we dont copy meta file type, it will be the same for all "new" objects + auto oh = _fsb_dst.newObject(ByteSpan{e.e.get().v}); + + if (!static_cast(oh)) { + // already exists + return false; + } + + { // sync meta + // some hardcoded ehpemeral (besides mft/id) + oh.emplace_or_replace(e.e.get_or_emplace()); + oh.emplace_or_replace(e.e.get_or_emplace()); + + // serializable + for (const auto& [type, fn] : _os_src.registry().ctx().get>()._serl) { + //if (!e.e.registry()->storage(type)->contains(e.e)) { + //continue; + //} + + // this is hacky but we serialize and then deserialize the component + // raw copy might be better in the future + nlohmann::json tmp_j; + if (fn(e.e, tmp_j)) { + _os_dst.registry().ctx().get>()._deserl.at(type)(oh, tmp_j); + } + } + } + + static_cast(_fsb_dst).write(oh, ByteSpan{tmp_buffer}); + + //assert(std::filesystem::file_size(e.e.get().path) == std::filesystem::file_size(oh.get().path)); + + return false; + } + + bool onEvent(const ObjectStore::Events::ObjectUpdate&) override { + std::cerr << "Update called\n"; + assert(false); + return false; + } + } el { + os_src, + fsb_src, + os_dst, + fsb_dst, + }; + + // perform scan (which triggers events) + fsb_dst.scanAsync(); // fill with existing? + fsb_src.scanAsync(); // the scan + + // done + return 0; +} +