1 //
2 // inmemory_filesystem.cpp
3 //
4 // Copyright © 2024 Apple Inc. All rights reserved.
5 //
6 // Please refer to the license found in the LICENSE file in the root directory of the source tree.
7
8 #include "inmemory_filesystem.hpp"
9
10 #include <assert.h>
11 #include <fstream>
12 #include <iostream>
13 #include <sstream>
14
15 #if __has_include(<filesystem>)
16 #include <filesystem>
17 #elif __has_include(<experimental/filesystem>)
18 #include <experimental/filesystem>
19 namespace std {
20 namespace filesystem = std::experimental::filesystem;
21 }
22 #endif
23
24 #include "range.hpp"
25 #include "reversed_memory_stream.hpp"
26
27 namespace {
28 using namespace inmemoryfs;
29
30 class InMemoryFileNode : public InMemoryFileSystem::InMemoryNode {
31 public:
InMemoryFileNode(std::string name,InMemoryFileSystem::Attributes attributes,std::shared_ptr<MemoryBuffer> buffer)32 InMemoryFileNode(std::string name,
33 InMemoryFileSystem::Attributes attributes,
34 std::shared_ptr<MemoryBuffer> buffer) noexcept
35 : InMemoryNode(std::move(name), std::move(attributes), InMemoryNode::Kind::File), buffer_(std::move(buffer)) { }
36
37 InMemoryFileNode(InMemoryFileNode const&) = delete;
38 InMemoryFileNode& operator=(InMemoryFileNode const&) = delete;
39
getBuffer() const40 inline std::shared_ptr<MemoryBuffer> getBuffer() const noexcept { return buffer_; }
41
42 private:
43 const std::shared_ptr<MemoryBuffer> buffer_;
44 };
45
46 class InMemoryDirectoryNode : public InMemoryFileSystem::InMemoryNode {
47 public:
48 using ItemsType = std::unordered_map<std::string, std::unique_ptr<InMemoryNode>>;
49
InMemoryDirectoryNode(std::string name,InMemoryFileSystem::Attributes attributes,ItemsType items)50 InMemoryDirectoryNode(std::string name, InMemoryFileSystem::Attributes attributes, ItemsType items) noexcept
51 : InMemoryNode(std::move(name), std::move(attributes), InMemoryNode::Kind::Directory),
52 items_(std::move(items)) { }
53
InMemoryDirectoryNode(std::string name,InMemoryFileSystem::Attributes attributes)54 InMemoryDirectoryNode(std::string name, InMemoryFileSystem::Attributes attributes) noexcept
55 : InMemoryNode(std::move(name), std::move(attributes), InMemoryNode::Kind::Directory), items_(ItemsType()) { }
56
57 InMemoryDirectoryNode(InMemoryDirectoryNode const&) = delete;
58 InMemoryDirectoryNode& operator=(InMemoryDirectoryNode const&) = delete;
59
add_item(const std::string & name,std::unique_ptr<InMemoryNode> node)60 inline void add_item(const std::string& name, std::unique_ptr<InMemoryNode> node) noexcept {
61 items_[name] = std::move(node);
62 }
63
remove_item(const std::string & name)64 inline void remove_item(const std::string& name) noexcept { items_.erase(name); }
65
get_items() const66 inline const ItemsType& get_items() const noexcept { return items_; }
67
contains(const std::string & key) const68 inline bool contains(const std::string& key) const noexcept { return items_.find(key) != items_.end(); }
69
get_item(const std::string & key)70 InMemoryNode* get_item(const std::string& key) noexcept {
71 auto it = items_.find(key);
72 if (it == items_.end()) {
73 return nullptr;
74 }
75
76 return it->second.get();
77 }
78
rename_item(const std::string & old_name,const std::string & new_name)79 InMemoryNode* rename_item(const std::string& old_name, const std::string& new_name) noexcept {
80 auto it = items_.find(old_name);
81 if (it == items_.end()) {
82 return nullptr;
83 }
84
85 auto node = std::move(it->second);
86 auto ptr = node.get();
87 items_.erase(old_name);
88 items_.emplace(new_name, std::move(node));
89
90 return ptr;
91 }
92
93 private:
94 ItemsType items_;
95 };
96
is_last(Iter it,const Container & container)97 template <typename Iter, typename Container> inline bool is_last(Iter it, const Container& container) {
98 return (it != container.end()) && (next(it) == container.end());
99 }
100
align(size_t length,size_t alignment)101 inline size_t align(size_t length, size_t alignment) { return ((length + (alignment - 1)) / alignment) * alignment; }
102
get_node(InMemoryFileSystem::InMemoryNode * node,std::vector<std::string>::const_iterator path_start,std::vector<std::string>::const_iterator path_end)103 InMemoryFileSystem::InMemoryNode* get_node(InMemoryFileSystem::InMemoryNode* node,
104 std::vector<std::string>::const_iterator path_start,
105 std::vector<std::string>::const_iterator path_end) {
106 for (auto it = path_start; it != path_end; ++it) {
107 if (!node) {
108 return nullptr;
109 }
110 const std::string& component = *it;
111 InMemoryDirectoryNode* directory_node = static_cast<InMemoryDirectoryNode*>(node);
112 node = directory_node->get_item(component);
113 }
114
115 return node;
116 }
117
118
toTime(const std::string & str)119 time_t toTime(const std::string& str) {
120 constexpr auto format = "%Y-%m-%dT%TZ";
121 time_t time = (time_t)(-1);
122 std::stringstream stream(str);
123 stream >> std::get_time(gmtime(&time), format);
124 return time;
125 }
126
toTime(TP tp)127 template <typename TP> std::time_t toTime(TP tp) {
128 using namespace std::chrono;
129 auto sctp = time_point_cast<system_clock::duration>(tp - TP::clock::now() + system_clock::now());
130 return system_clock::to_time_t(sctp);
131 }
132
get_file_attributes(const std::filesystem::path & path)133 InMemoryFileSystem::Attributes get_file_attributes(const std::filesystem::path& path) {
134 auto attributes = InMemoryFileSystem::Attributes();
135 auto modificationTime = toTime(std::filesystem::last_write_time(path));
136 attributes.modificationTime = modificationTime;
137
138 return attributes;
139 }
140
to_memory_buffer_read_option(InMemoryFileSystem::FileLoadOption option)141 MemoryBuffer::ReadOption to_memory_buffer_read_option(InMemoryFileSystem::FileLoadOption option) {
142 switch (option) {
143 case InMemoryFileSystem::FileLoadOption::Malloc:
144 return MemoryBuffer::ReadOption::Malloc;
145
146 case InMemoryFileSystem::FileLoadOption::MMap:
147 return MemoryBuffer::ReadOption::MMap;
148
149 case InMemoryFileSystem::FileLoadOption::LazyMMap:
150 return MemoryBuffer::ReadOption::LazyMMap;
151 }
152 }
153
154 std::unique_ptr<InMemoryFileSystem::InMemoryNode>
make_file_node(const std::filesystem::path & path,InMemoryFileSystem::FileLoadOption option,std::error_code & error)155 make_file_node(const std::filesystem::path& path, InMemoryFileSystem::FileLoadOption option, std::error_code& error) {
156 auto name = path.filename().string();
157 auto file_path = path.string();
158
159
160 auto buffer = MemoryBuffer::read_file_content(file_path, to_memory_buffer_read_option(option), error);
161 if (error) {
162 return nullptr;
163 }
164
165 auto attributes = get_file_attributes(path);
166 return std::make_unique<InMemoryFileNode>(std::move(name), std::move(attributes), std::move(buffer));
167 }
168
make_directory_node(const std::filesystem::path & path,InMemoryFileSystem::FileLoadOption option,std::error_code & error)169 std::unique_ptr<InMemoryFileSystem::InMemoryNode> make_directory_node(const std::filesystem::path& path,
170 InMemoryFileSystem::FileLoadOption option,
171 std::error_code& error) {
172 auto name = path.filename();
173 std::unordered_map<std::string, std::unique_ptr<InMemoryFileSystem::InMemoryNode>> items = {};
174 for (const std::filesystem::directory_entry& entry: std::filesystem::directory_iterator(path)) {
175 if (!entry.exists()) {
176 continue;
177 }
178
179 auto itemPath = std::filesystem::canonical(entry.path());
180 auto itemName = itemPath.filename().string();
181 std::unique_ptr<InMemoryFileSystem::InMemoryNode> node;
182 if (entry.is_directory()) {
183 node = make_directory_node(itemPath, option, error);
184 } else if (!entry.is_directory()) {
185 node = make_file_node(itemPath, option, error);
186 }
187
188 if (node) {
189 items[itemName] = std::move(node);
190 }
191 }
192 auto attributes = get_file_attributes(path);
193 return std::make_unique<InMemoryDirectoryNode>(std::move(name), std::move(attributes), std::move(items));
194 }
195
196 std::unique_ptr<InMemoryFileSystem::InMemoryNode>
make_node(const std::filesystem::path & path,InMemoryFileSystem::FileLoadOption option,std::error_code & error)197 make_node(const std::filesystem::path& path, InMemoryFileSystem::FileLoadOption option, std::error_code& error) {
198 auto status = std::filesystem::exists(path, error);
199 if (!status || error) {
200 return nullptr;
201 }
202
203 auto name = path.filename();
204 bool isDirectory = std::filesystem::is_directory(path, error);
205 if (error) {
206 return nullptr;
207 }
208
209 if (isDirectory) {
210 return make_directory_node(path, option, error);
211 }
212
213 return make_file_node(path, option, error);
214 }
215
216 bool write_node(InMemoryFileSystem::InMemoryNode* node,
217 const std::filesystem::path& dst_path,
218 bool recursive,
219 std::error_code& error);
220
write_file_node(InMemoryFileNode * node,const std::filesystem::path & dst_path,std::error_code & error)221 bool write_file_node(InMemoryFileNode* node, const std::filesystem::path& dst_path, std::error_code& error) {
222 std::filesystem::path file_path = dst_path;
223 file_path.append(node->name());
224 std::ofstream stream;
225 stream.open(file_path, std::ofstream::out);
226 if (!stream.good()) {
227 error = std::error_code(errno, std::system_category());
228 return false;
229 }
230 auto buffer = node->getBuffer().get();
231 if (buffer->size() > 0) {
232 if (!buffer->load(error)) {
233 return false;
234 }
235 char* bufferPtr = static_cast<char*>(buffer->data());
236 stream.write(bufferPtr, static_cast<std::streamsize>(buffer->size()));
237 }
238 if (!stream.good()) {
239 error = std::error_code(errno, std::system_category());
240 }
241 stream.close();
242
243 return !error;
244 }
245
write_directory_node(InMemoryDirectoryNode * node,const std::filesystem::path & dst_path,bool recursive,std::error_code & error)246 bool write_directory_node(InMemoryDirectoryNode* node,
247 const std::filesystem::path& dst_path,
248 bool recursive,
249 std::error_code& error) {
250 std::filesystem::path dir_path = dst_path;
251 dir_path.append(node->name());
252 if (!std::filesystem::create_directory(dir_path, error)) {
253 return false;
254 }
255
256 for (const auto& [_, node_2]: node->get_items()) {
257 if (node_2.get()->isDirectory() && !recursive) {
258 continue;
259 }
260 if (!write_node(node_2.get(), dir_path, recursive, error)) {
261 return false;
262 }
263 }
264
265 return true;
266 }
267
write_node(InMemoryFileSystem::InMemoryNode * node,const std::filesystem::path & dst_path,bool recursive,std::error_code & error)268 bool write_node(InMemoryFileSystem::InMemoryNode* node,
269 const std::filesystem::path& dst_path,
270 bool recursive,
271 std::error_code& error) {
272 switch (node->kind()) {
273 case InMemoryFileSystem::InMemoryNode::Kind::Directory:
274 return write_directory_node(static_cast<InMemoryDirectoryNode*>(node), dst_path, recursive, error);
275
276 case InMemoryFileSystem::InMemoryNode::Kind::File:
277 return write_file_node(static_cast<InMemoryFileNode*>(node), dst_path, error);
278 }
279 }
280
281 struct Attributes {
282 time_t creation_time;
283 time_t modification_time;
284
Attributes__anon9cfc14fc0111::Attributes285 inline Attributes() noexcept : creation_time(time(0)), modification_time(time(0)) { }
286 };
287
288 struct FlattenedInMemoryNode {
289 InMemoryNodeMetadata metadata;
290 InMemoryFileNode* file_node = nullptr;
291
FlattenedInMemoryNode__anon9cfc14fc0111::FlattenedInMemoryNode292 FlattenedInMemoryNode(InMemoryNodeMetadata metadata) noexcept : metadata(std::move(metadata)) { }
293
FlattenedInMemoryNode__anon9cfc14fc0111::FlattenedInMemoryNode294 FlattenedInMemoryNode() noexcept { }
295
296 static std::vector<FlattenedInMemoryNode> flatten(InMemoryFileSystem::InMemoryNode* node,
297 size_t alignment) noexcept;
298
299 static std::unique_ptr<InMemoryFileSystem::InMemoryNode>
300 unflatten(const std::vector<FlattenedInMemoryNode>& nodes, const std::shared_ptr<MemoryBuffer>& data) noexcept;
301 };
302
next_offset_to_write(const std::vector<FlattenedInMemoryNode> & nodes)303 size_t next_offset_to_write(const std::vector<FlattenedInMemoryNode>& nodes) noexcept {
304 for (auto it = nodes.rbegin(); it != nodes.rend(); ++it) {
305 const auto& metadata = it->metadata;
306 if (metadata.kind == static_cast<int>(InMemoryFileSystem::InMemoryNode::Kind::File)) {
307 return metadata.data_region.length();
308 }
309 }
310
311 return 0;
312 }
313
populate(InMemoryFileSystem::InMemoryNode * node,std::unordered_map<InMemoryFileSystem::InMemoryNode *,size_t> & node_to_index_map,size_t alignment,std::vector<FlattenedInMemoryNode> & result)314 void populate(InMemoryFileSystem::InMemoryNode* node,
315 std::unordered_map<InMemoryFileSystem::InMemoryNode*, size_t>& node_to_index_map,
316 size_t alignment,
317 std::vector<FlattenedInMemoryNode>& result) noexcept {
318 FlattenedInMemoryNode flattened_node;
319 auto& flattened_node_metadata = flattened_node.metadata;
320 flattened_node_metadata.name = node->name();
321 flattened_node_metadata.kind = static_cast<size_t>(node->kind());
322 switch (node->kind()) {
323 case InMemoryFileSystem::InMemoryNode::Kind::File: {
324 size_t index = result.size();
325 InMemoryFileNode* file_node = static_cast<InMemoryFileNode*>(node);
326 size_t offset = align(next_offset_to_write(result), alignment);
327 auto buffer = file_node->getBuffer();
328 size_t size = buffer->size();
329 flattened_node.metadata.data_region = Range(offset, size);
330 flattened_node.file_node = file_node;
331 node_to_index_map[node] = index;
332 break;
333 }
334 case InMemoryFileSystem::InMemoryNode::Kind::Directory: {
335 InMemoryDirectoryNode* directory_node = static_cast<InMemoryDirectoryNode*>(node);
336 for (const auto& [key, item]: directory_node->get_items()) {
337 populate(item.get(), node_to_index_map, alignment, result);
338 flattened_node_metadata.child_name_to_indices_map[key] = node_to_index_map[item.get()];
339 }
340 node_to_index_map[node] = result.size();
341 break;
342 }
343 }
344
345 result.emplace_back(std::move(flattened_node));
346 }
347
flatten(InMemoryFileSystem::InMemoryNode * node,size_t alignment)348 std::vector<FlattenedInMemoryNode> FlattenedInMemoryNode::flatten(InMemoryFileSystem::InMemoryNode* node,
349 size_t alignment) noexcept {
350 std::unordered_map<InMemoryFileSystem::InMemoryNode*, size_t> node_to_index_map;
351 std::vector<FlattenedInMemoryNode> result;
352 populate(node, node_to_index_map, alignment, result);
353
354 return result;
355 }
356
357 std::unique_ptr<InMemoryFileSystem::InMemoryNode>
unflatten(const std::vector<FlattenedInMemoryNode> & flattened_nodes,const std::shared_ptr<MemoryBuffer> & buffer)358 FlattenedInMemoryNode::unflatten(const std::vector<FlattenedInMemoryNode>& flattened_nodes,
359 const std::shared_ptr<MemoryBuffer>& buffer) noexcept {
360 if (flattened_nodes.size() == 0) {
361 return nullptr;
362 }
363
364 std::vector<std::unique_ptr<InMemoryFileSystem::InMemoryNode>> nodes;
365 nodes.reserve(flattened_nodes.size());
366 for (size_t index = 0; index < flattened_nodes.size(); index++) {
367 const FlattenedInMemoryNode& flattened_node = flattened_nodes[index];
368 const auto& flattened_node_metadata = flattened_node.metadata;
369 auto name = flattened_node_metadata.name;
370 auto attributes = InMemoryFileSystem::Attributes();
371 switch (static_cast<InMemoryFileSystem::InMemoryNode::Kind>(flattened_node_metadata.kind)) {
372 case InMemoryFileSystem::InMemoryNode::Kind::File: {
373 auto region = flattened_node_metadata.data_region;
374 std::shared_ptr sliced_buffer = buffer->slice(region);
375 if (!sliced_buffer) {
376 return nullptr;
377 }
378 auto file_node = std::make_unique<InMemoryFileNode>(
379 std::move(name), std::move(attributes), std::move(sliced_buffer));
380 nodes.emplace_back(std::move(file_node));
381 break;
382 }
383 case InMemoryFileSystem::InMemoryNode::Kind::Directory: {
384 std::unordered_map<std::string, std::unique_ptr<InMemoryFileSystem::InMemoryNode>> items;
385 items.reserve(flattened_node_metadata.child_name_to_indices_map.size());
386 for (const auto& [name_2, index_2]: flattened_node_metadata.child_name_to_indices_map) {
387 auto moveIt = std::make_move_iterator(nodes.begin() + index_2);
388 items[name_2] = *moveIt;
389 }
390 auto directory_node =
391 std::make_unique<InMemoryDirectoryNode>(std::move(name), std::move(attributes), std::move(items));
392 nodes.emplace_back(std::move(directory_node));
393 break;
394 }
395 }
396 }
397
398 return std::move(nodes.back());
399 }
400
get_metadatas(std::vector<FlattenedInMemoryNode> flattened_nodes)401 InMemoryFileSystemMetadata get_metadatas(std::vector<FlattenedInMemoryNode> flattened_nodes) {
402 std::vector<InMemoryNodeMetadata> node_metadatas;
403 node_metadatas.reserve(flattened_nodes.size());
404 std::transform(std::make_move_iterator(flattened_nodes.begin()),
405 std::make_move_iterator(flattened_nodes.end()),
406 std::back_inserter(node_metadatas),
407 [](FlattenedInMemoryNode&& flattened_node) { return std::move(flattened_node.metadata); });
408
409 return InMemoryFileSystemMetadata { .nodes = std::move(node_metadatas) };
410 }
411
get_flattened_nodes(std::vector<InMemoryNodeMetadata> node_metadatas)412 std::vector<FlattenedInMemoryNode> get_flattened_nodes(std::vector<InMemoryNodeMetadata> node_metadatas) {
413 std::vector<FlattenedInMemoryNode> flattened_nodes;
414 flattened_nodes.reserve(node_metadatas.size());
415 std::transform(
416 std::make_move_iterator(node_metadatas.begin()),
417 std::make_move_iterator(node_metadatas.end()),
418 std::back_inserter(flattened_nodes),
419 [](InMemoryNodeMetadata&& node_metadata) { return FlattenedInMemoryNode(std::move(node_metadata)); });
420
421 return flattened_nodes;
422 }
423
fill_stream(std::vector<uint8_t> & buffer,std::ostream & stream,size_t size)424 bool fill_stream(std::vector<uint8_t>& buffer, std::ostream& stream, size_t size) {
425 if (size == 0) {
426 return true;
427 }
428
429 size_t n = size / buffer.size();
430 for (size_t i = 0; i < n; i++) {
431 if (!stream.write(reinterpret_cast<char*>(buffer.data()), buffer.size()).good()) {
432 return false;
433 }
434 }
435
436 size_t rem = size % buffer.size();
437 if (rem > 0) {
438 if (!stream.write(reinterpret_cast<char*>(buffer.data()), rem).good()) {
439 return false;
440 }
441 }
442
443 return true;
444 }
445
446 } // namespace
447
448 namespace inmemoryfs {
449
message(int code) const450 std::string InMemoryFileSystem::ErrorCategory::message(int code) const {
451 switch (static_cast<ErrorCode>(code)) {
452 case ErrorCode::DirectoryExists:
453 return "The item at the path is a directory.";
454 case ErrorCode::ItemNotFound:
455 return "Path does not exist.";
456 case ErrorCode::ItemExists:
457 return "Item already exists at the path.";
458 case ErrorCode::DirectoryExpected:
459 return "The item at the path is not a directory";
460 case ErrorCode::FileExpected:
461 return "The item at the path is not a file";
462 }
463 }
464
InMemoryFileSystem(std::string name)465 InMemoryFileSystem::InMemoryFileSystem(std::string name) noexcept
466 : root_(std::make_unique<InMemoryDirectoryNode>(std::move(name), Attributes())) { }
467
is_directory(const std::vector<std::string> & canonical_path)468 bool InMemoryFileSystem::is_directory(const std::vector<std::string>& canonical_path) noexcept {
469 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
470 return node && node->isDirectory();
471 }
472
is_file(const std::vector<std::string> & canonical_path)473 bool InMemoryFileSystem::is_file(const std::vector<std::string>& canonical_path) noexcept {
474 auto node = get_node(root_.get(), canonical_path.begin(), canonical_path.end());
475 return node && node->isFile();
476 }
477
exists(const std::vector<std::string> & canonical_path) const478 bool InMemoryFileSystem::exists(const std::vector<std::string>& canonical_path) const noexcept {
479 auto node = get_node(root_.get(), canonical_path.begin(), canonical_path.end());
480 return node != nullptr;
481 }
482
get_item_paths(const std::vector<std::string> & canonical_path,std::error_code & error) const483 std::vector<std::vector<std::string>> InMemoryFileSystem::get_item_paths(const std::vector<std::string>& canonical_path,
484 std::error_code& error) const noexcept {
485 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
486 if (node == nullptr) {
487 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
488 return {};
489 }
490
491 if (node->isFile()) {
492 error = InMemoryFileSystem::ErrorCode::DirectoryExpected;
493 return {};
494 }
495
496 auto directory_node = static_cast<InMemoryDirectoryNode*>(node);
497 const auto& items = directory_node->get_items();
498 std::vector<std::vector<std::string>> result;
499 result.reserve(items.size());
500 for (const auto& [component, _]: items) {
501 auto components = canonical_path;
502 components.emplace_back(component);
503 result.emplace_back(std::move(components));
504 }
505
506 return result;
507 }
508
509 std::optional<InMemoryFileSystem::Attributes>
get_attributes(const std::vector<std::string> & canonical_path,std::error_code & error) const510 InMemoryFileSystem::get_attributes(const std::vector<std::string>& canonical_path,
511 std::error_code& error) const noexcept {
512 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
513 if (node == nullptr) {
514 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
515 return std::nullopt;
516 }
517
518 return node->attributes();
519 }
520
get_file_content(const std::vector<std::string> & canonical_path,std::error_code & error) const521 std::shared_ptr<MemoryBuffer> InMemoryFileSystem::get_file_content(const std::vector<std::string>& canonical_path,
522 std::error_code& error) const noexcept {
523 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
524 if (node == nullptr) {
525 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
526 return nullptr;
527 }
528 if (node->isDirectory()) {
529 error = InMemoryFileSystem::ErrorCode::FileExpected;
530 return nullptr;
531 }
532
533 InMemoryFileNode* file_node = static_cast<InMemoryFileNode*>(node);
534 return file_node->getBuffer();
535 }
536
make_directory(const std::vector<std::string> & canonical_path,Attributes attributes,bool create_intermediate_directories,std::error_code & error)537 bool InMemoryFileSystem::make_directory(const std::vector<std::string>& canonical_path,
538 Attributes attributes,
539 bool create_intermediate_directories,
540 std::error_code& error) noexcept {
541 if (canonical_path.size() == 0) {
542 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
543 return false;
544 }
545
546 auto directory_node = static_cast<InMemoryDirectoryNode*>(root());
547 for (auto it = canonical_path.begin(); it != canonical_path.end(); ++it) {
548 auto node = directory_node->get_item(*it);
549 bool createDirectory = create_intermediate_directories || is_last(it, canonical_path);
550 if (node == nullptr && createDirectory) {
551 auto child_directory_node = std::make_unique<InMemoryDirectoryNode>(*it, attributes);
552 directory_node->add_item(*it, std::move(child_directory_node));
553 directory_node = static_cast<InMemoryDirectoryNode*>(directory_node->get_item(*it));
554 } else if (node != nullptr && node->isDirectory()) {
555 directory_node = static_cast<InMemoryDirectoryNode*>(node);
556 } else {
557 if (node == nullptr) {
558 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
559 } else if (node->isFile()) {
560 error = InMemoryFileSystem::ErrorCode::DirectoryExpected;
561 }
562 return false;
563 }
564 }
565
566 return true;
567 }
568
make_file(const std::vector<std::string> & canonical_path,std::shared_ptr<MemoryBuffer> buffer,Attributes attributes,bool overwrite,std::error_code & error)569 bool InMemoryFileSystem::make_file(const std::vector<std::string>& canonical_path,
570 std::shared_ptr<MemoryBuffer> buffer,
571 Attributes attributes,
572 bool overwrite,
573 std::error_code& error) noexcept {
574 if (canonical_path.size() == 0) {
575 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
576 return false;
577 }
578
579 auto node = get_node(root(), canonical_path.begin(), std::prev(canonical_path.end()));
580 if (!node) {
581 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
582 return false;
583 }
584
585 if (!node->isDirectory()) {
586 error = InMemoryFileSystem::ErrorCode::DirectoryExpected;
587 return false;
588 }
589
590 InMemoryDirectoryNode* directory_node = static_cast<InMemoryDirectoryNode*>(node);
591 if (directory_node->contains(canonical_path.back()) && !overwrite) {
592 error = InMemoryFileSystem::ErrorCode::ItemExists;
593 return false;
594 }
595
596 auto name = canonical_path.back();
597 auto file_node = std::make_unique<InMemoryFileNode>(std::move(name), std::move(attributes), std::move(buffer));
598 directory_node->add_item(canonical_path.back(), std::move(file_node));
599
600 return true;
601 }
602
remove_item(const std::vector<std::string> & canonical_path,std::error_code & error)603 bool InMemoryFileSystem::remove_item(const std::vector<std::string>& canonical_path, std::error_code& error) noexcept {
604 if (canonical_path.size() == 0) {
605 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
606 return false;
607 }
608
609 auto node = get_node(root(), canonical_path.begin(), std::prev(canonical_path.end()));
610 if (!node->isDirectory()) {
611 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
612 return false;
613 }
614
615 InMemoryDirectoryNode* directory_node = static_cast<InMemoryDirectoryNode*>(node);
616 if (!directory_node->contains(canonical_path.back())) {
617 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
618 return false;
619 }
620 directory_node->remove_item(canonical_path.back());
621
622 return true;
623 }
624
rename_item(const std::vector<std::string> & canonical_path,const std::string & name,std::error_code & error)625 bool InMemoryFileSystem::rename_item(const std::vector<std::string>& canonical_path,
626 const std::string& name,
627 std::error_code& error) noexcept {
628
629 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
630 if (!node) {
631 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
632 return false;
633 }
634
635 auto parent_node =
636 canonical_path.size() > 0 ? get_node(root(), canonical_path.begin(), std::prev(canonical_path.end())) : nullptr;
637 if (parent_node && parent_node->isDirectory()) {
638 InMemoryDirectoryNode* parent_directory_node = static_cast<InMemoryDirectoryNode*>(parent_node);
639 if (parent_directory_node->contains(name)) {
640 error = InMemoryFileSystem::ErrorCode::ItemExists;
641 return false;
642 }
643
644 parent_directory_node->rename_item(node->name(), name);
645 } else {
646 node->set_name(name);
647 }
648
649 return true;
650 }
651
set_attributes(const std::vector<std::string> & canonical_path,Attributes attributes,std::error_code & error)652 bool InMemoryFileSystem::set_attributes(const std::vector<std::string>& canonical_path,
653 Attributes attributes,
654 std::error_code& error) noexcept {
655 if (canonical_path.size() == 0) {
656 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
657 return false;
658 }
659
660 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
661 if (!node) {
662 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
663 return false;
664 }
665
666 node->set_attributes(std::move(attributes));
667 return true;
668 }
669
write_item_to_disk(const std::vector<std::string> & canonical_path,const std::string & dst_path,bool recursive,std::error_code & error) const670 bool InMemoryFileSystem::write_item_to_disk(const std::vector<std::string>& canonical_path,
671 const std::string& dst_path,
672 bool recursive,
673 std::error_code& error) const noexcept {
674 std::filesystem::path dst_file_path(dst_path);
675 auto status = std::filesystem::exists(dst_file_path, error);
676 if (!status || error) {
677 return false;
678 }
679
680 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
681 if (!node) {
682 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
683 return false;
684 }
685
686 bool result = write_node(node, dst_file_path, recursive, error);
687 if (!result) {
688 auto rootPath = dst_path;
689 rootPath.append(root()->name());
690 std::filesystem::remove_all(rootPath, error);
691 }
692
693 return result;
694 }
695
make_from_directory(const std::string & path,FileLoadOption option,std::error_code & error)696 std::unique_ptr<InMemoryFileSystem> InMemoryFileSystem::make_from_directory(const std::string& path,
697 FileLoadOption option,
698 std::error_code& error) noexcept {
699 std::filesystem::path file_path(path);
700 auto status = std::filesystem::exists(file_path, error);
701 if (error) {
702 return nullptr;
703 }
704
705 if (!status) {
706 error = InMemoryFileSystem::ErrorCode::ItemNotFound;
707 return nullptr;
708 }
709
710 auto node = make_node(file_path, option, error);
711 if (!node) {
712 return nullptr;
713 }
714
715 switch (node->kind()) {
716 case InMemoryFileSystem::InMemoryNode::Kind::Directory: {
717 auto fs = std::make_unique<InMemoryFileSystem>(std::move(node));
718 return fs;
719 }
720 case InMemoryFileSystem::InMemoryNode::Kind::File: {
721 auto fs = std::make_unique<InMemoryFileSystem>();
722 auto rootNode = static_cast<InMemoryDirectoryNode*>(fs->root());
723 rootNode->add_item(file_path.filename().string(), std::move(node));
724 return fs;
725 }
726 }
727 }
728
serialize(const std::vector<std::string> & canonical_path,size_t alignment,const MetadataWriter & metadata_writer,std::ostream & stream,std::error_code & error) const729 bool InMemoryFileSystem::serialize(const std::vector<std::string>& canonical_path,
730 size_t alignment,
731 const MetadataWriter& metadata_writer,
732 std::ostream& stream,
733 std::error_code& error) const noexcept {
734 assert(alignment > 0);
735 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
736 if (!node) {
737 return true;
738 }
739
740 static constexpr size_t buffer_size = 512;
741 std::vector<uint8_t> empty_buffer(buffer_size, 0);
742 auto flattened_nodes = FlattenedInMemoryNode::flatten(node, alignment);
743 size_t write_pos = 0;
744 for (const auto& flattened_node: flattened_nodes) {
745 if (flattened_node.file_node == nullptr) {
746 continue;
747 }
748
749 const auto& flattened_node_metadata = flattened_node.metadata;
750 auto range = flattened_node_metadata.data_region;
751 auto buffer = flattened_node.file_node->getBuffer();
752 if (!buffer->load(error)) {
753 return false;
754 }
755
756 auto start = static_cast<char*>(buffer->data());
757 assert(range.offset >= write_pos);
758 if (!fill_stream(empty_buffer, stream, range.offset - write_pos)) {
759 error = std::error_code(errno, std::system_category());
760 return false;
761 }
762
763 if (!stream.write(start, range.size).good()) {
764 error = std::error_code(errno, std::system_category());
765 return false;
766 }
767
768 write_pos = std::max(write_pos, range.length());
769 }
770
771 size_t metadata_write_pos = align(write_pos, alignment);
772 if (!fill_stream(empty_buffer, stream, metadata_write_pos - write_pos)) {
773 error = std::error_code(errno, std::system_category());
774 return false;
775 }
776
777 auto fs_metadata = get_metadatas(std::move(flattened_nodes));
778 // Serialize metadata at the end of the stream.
779 if (!metadata_writer(fs_metadata, stream)) {
780 error = std::error_code(errno, std::system_category());
781 return false;
782 }
783
784 return true;
785 }
786
get_buffer_size_for_serialization(const std::vector<std::string> & canonical_path,size_t alignment,const MetadataWriter & metadata_writer) const787 size_t InMemoryFileSystem::get_buffer_size_for_serialization(const std::vector<std::string>& canonical_path,
788 size_t alignment,
789 const MetadataWriter& metadata_writer) const noexcept {
790 assert(alignment > 0);
791 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
792 if (!node) {
793 return 0;
794 }
795
796 auto flattened_nodes = FlattenedInMemoryNode::flatten(node, alignment);
797 size_t length = 0;
798 size_t change = 0;
799 for (auto& flattened_node: flattened_nodes) {
800 if (flattened_node.file_node == nullptr) {
801 continue;
802 }
803
804 auto& data_region = flattened_node.metadata.data_region;
805 auto offset_range = flattened_node.file_node->getBuffer()->get_offset_range(data_region.offset + change);
806 size_t max_offset = offset_range.second;
807 change += (max_offset - data_region.offset);
808 data_region.offset = max_offset;
809 length = data_region.length();
810 }
811
812 length = align(length, alignment);
813 auto fs_metadata = get_metadatas(std::move(flattened_nodes));
814 std::stringstream stream;
815 metadata_writer(fs_metadata, stream);
816 assert(stream.good());
817 length += stream.str().length();
818
819 return length;
820 }
821
822
serialize(const std::vector<std::string> & canonical_path,size_t alignment,const MetadataWriterInMemory & metadata_writer,void * dst,std::error_code & error) const823 bool InMemoryFileSystem::serialize(const std::vector<std::string>& canonical_path,
824 size_t alignment,
825 const MetadataWriterInMemory& metadata_writer,
826 void* dst,
827 std::error_code& error) const noexcept {
828 assert(alignment > 0);
829 auto node = get_node(root(), canonical_path.begin(), canonical_path.end());
830 if (!node) {
831 return true;
832 }
833
834 uint8_t* ptr = static_cast<uint8_t*>(dst);
835 size_t write_pos = 0;
836 ssize_t change = 0;
837 auto flattened_nodes = FlattenedInMemoryNode::flatten(node, alignment);
838 for (auto& flattened_node: flattened_nodes) {
839 if (flattened_node.file_node == nullptr) {
840 continue;
841 }
842
843 auto& data_region = flattened_node.metadata.data_region;
844 auto buffer = flattened_node.file_node->getBuffer();
845 // Get the revised range that must be used for writing the buffer content.
846 Range revised_data_region =
847 buffer->get_revised_range_for_writing(dst, Range(data_region.offset + change, data_region.size));
848 if (!buffer->write(ptr, revised_data_region.offset, error)) {
849 return false;
850 }
851
852 change += (revised_data_region.offset - data_region.offset);
853 // update data region.
854 data_region = revised_data_region;
855 write_pos = std::max(write_pos, data_region.length());
856 }
857
858 size_t metadata_write_pos = align(write_pos, alignment);
859 auto fs_metadata = get_metadatas(std::move(flattened_nodes));
860 // Serialize metadata at the end of the stream.
861 metadata_writer(fs_metadata, ptr + metadata_write_pos);
862 return true;
863 }
864
865 std::unique_ptr<InMemoryFileSystem>
make_from_buffer(const std::shared_ptr<MemoryBuffer> & buffer,const MetadataReader & metadata_reader)866 InMemoryFileSystem::make_from_buffer(const std::shared_ptr<MemoryBuffer>& buffer,
867 const MetadataReader& metadata_reader) noexcept {
868 // read metadata from the end of the stream
869 auto istream = ReversedIMemoryStream(buffer);
870 auto fs_metadata = metadata_reader(istream);
871 if (!fs_metadata) {
872 return nullptr;
873 }
874
875 auto flattened_nodes = get_flattened_nodes(std::move(fs_metadata.value().nodes));
876 auto rootNode = FlattenedInMemoryNode::unflatten(flattened_nodes, buffer);
877 if (!rootNode) {
878 return nullptr;
879 }
880
881 return std::make_unique<InMemoryFileSystem>(std::move(rootNode));
882 }
883 } // namespace inmemoryfs
884