1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved. 2 3 Licensed under the Apache License, Version 2.0 (the "License"); 4 you may not use this file except in compliance with the License. 5 You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9 Unless required by applicable law or agreed to in writing, software 10 distributed under the License is distributed on an "AS IS" BASIS, 11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 See the License for the specific language governing permissions and 13 limitations under the License. 14 ==============================================================================*/ 15 #ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 16 #define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 17 18 #include <memory> 19 20 #include "tensorflow/c/experimental/filesystem/filesystem_interface.h" 21 #include "tensorflow/core/platform/file_system.h" 22 23 /// This file builds classes needed to hold a filesystem implementation in the 24 /// modular world. Once all TensorFlow filesystems are converted to use the 25 /// plugin based approach, this file will replace the one in core/platform and 26 /// the names will lose the `Modular` part. Until that point, the `Modular*` 27 /// classes here are experimental and subject to breaking changes. 28 /// For documentation on these methods, consult `core/platform/filesystem.h`. 29 30 namespace tensorflow { 31 32 // TODO(b/143949615): After all filesystems are converted, this file will be 33 // moved to core/platform, and this class can become a singleton and replace the 34 // need for `Env::Default()`. At that time, we might decide to remove the need 35 // for `Env::Default()` altogether, but that's a different project, not in 36 // scope for now. I'm just mentioning this here as that transition will mean 37 // removal of the registration part from `Env` and adding it here instead: we 38 // will need tables to hold for each scheme the function tables that implement 39 // the needed functionality instead of the current `FileSystemRegistry` code in 40 // `core/platform/env.cc`. 41 class ModularFileSystem final : public FileSystem { 42 public: ModularFileSystem(std::unique_ptr<TF_Filesystem> filesystem,std::unique_ptr<const TF_FilesystemOps> filesystem_ops,std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops,std::unique_ptr<const TF_WritableFileOps> writable_file_ops,std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> read_only_memory_region_ops,std::function<void * (size_t)> plugin_memory_allocate,std::function<void (void *)> plugin_memory_free)43 ModularFileSystem( 44 std::unique_ptr<TF_Filesystem> filesystem, 45 std::unique_ptr<const TF_FilesystemOps> filesystem_ops, 46 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops, 47 std::unique_ptr<const TF_WritableFileOps> writable_file_ops, 48 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> 49 read_only_memory_region_ops, 50 std::function<void*(size_t)> plugin_memory_allocate, 51 std::function<void(void*)> plugin_memory_free) 52 : filesystem_(std::move(filesystem)), 53 ops_(std::move(filesystem_ops)), 54 random_access_file_ops_(std::move(random_access_file_ops)), 55 writable_file_ops_(std::move(writable_file_ops)), 56 read_only_memory_region_ops_(std::move(read_only_memory_region_ops)), 57 plugin_memory_allocate_(std::move(plugin_memory_allocate)), 58 plugin_memory_free_(std::move(plugin_memory_free)) {} 59 ~ModularFileSystem()60 ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); } 61 62 TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT; 63 64 Status NewRandomAccessFile( 65 const std::string& fname, TransactionToken* token, 66 std::unique_ptr<RandomAccessFile>* result) override; 67 Status NewWritableFile(const std::string& fname, TransactionToken* token, 68 std::unique_ptr<WritableFile>* result) override; 69 Status NewAppendableFile(const std::string& fname, TransactionToken* token, 70 std::unique_ptr<WritableFile>* result) override; 71 Status NewReadOnlyMemoryRegionFromFile( 72 const std::string& fname, TransactionToken* token, 73 std::unique_ptr<ReadOnlyMemoryRegion>* result) override; 74 Status FileExists(const std::string& fname, TransactionToken* token) override; 75 bool FilesExist(const std::vector<std::string>& files, 76 TransactionToken* token, 77 std::vector<Status>* status) override; 78 Status GetChildren(const std::string& dir, TransactionToken* token, 79 std::vector<std::string>* result) override; 80 Status GetMatchingPaths(const std::string& pattern, TransactionToken* token, 81 std::vector<std::string>* results) override; 82 Status DeleteFile(const std::string& fname, TransactionToken* token) override; 83 Status DeleteRecursively(const std::string& dirname, TransactionToken* token, 84 int64_t* undeleted_files, 85 int64_t* undeleted_dirs) override; 86 Status DeleteDir(const std::string& dirname, 87 TransactionToken* token) override; 88 Status RecursivelyCreateDir(const std::string& dirname, 89 TransactionToken* token) override; 90 Status CreateDir(const std::string& dirname, 91 TransactionToken* token) override; 92 Status Stat(const std::string& fname, TransactionToken* token, 93 FileStatistics* stat) override; 94 Status IsDirectory(const std::string& fname, 95 TransactionToken* token) override; 96 Status GetFileSize(const std::string& fname, TransactionToken* token, 97 uint64* file_size) override; 98 Status RenameFile(const std::string& src, const std::string& target, 99 TransactionToken* token) override; 100 Status CopyFile(const std::string& src, const std::string& target, 101 TransactionToken* token) override; 102 std::string TranslateName(const std::string& name) const override; 103 void FlushCaches(TransactionToken* token) override; 104 Status SetOption(const std::string& name, 105 const std::vector<string>& values) override; 106 Status SetOption(const std::string& name, 107 const std::vector<int64_t>& values) override; 108 Status SetOption(const std::string& name, 109 const std::vector<double>& values) override; 110 111 private: 112 std::unique_ptr<TF_Filesystem> filesystem_; 113 std::unique_ptr<const TF_FilesystemOps> ops_; 114 std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops_; 115 std::unique_ptr<const TF_WritableFileOps> writable_file_ops_; 116 std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> 117 read_only_memory_region_ops_; 118 std::function<void*(size_t)> plugin_memory_allocate_; 119 std::function<void(void*)> plugin_memory_free_; 120 TF_DISALLOW_COPY_AND_ASSIGN(ModularFileSystem); 121 }; 122 123 class ModularRandomAccessFile final : public RandomAccessFile { 124 public: ModularRandomAccessFile(const std::string & filename,std::unique_ptr<TF_RandomAccessFile> file,const TF_RandomAccessFileOps * ops)125 ModularRandomAccessFile(const std::string& filename, 126 std::unique_ptr<TF_RandomAccessFile> file, 127 const TF_RandomAccessFileOps* ops) 128 : filename_(filename), file_(std::move(file)), ops_(ops) {} 129 ~ModularRandomAccessFile()130 ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); } 131 132 Status Read(uint64 offset, size_t n, StringPiece* result, 133 char* scratch) const override; 134 Status Name(StringPiece* result) const override; 135 136 private: 137 std::string filename_; 138 std::unique_ptr<TF_RandomAccessFile> file_; 139 const TF_RandomAccessFileOps* ops_; // not owned 140 TF_DISALLOW_COPY_AND_ASSIGN(ModularRandomAccessFile); 141 }; 142 143 class ModularWritableFile final : public WritableFile { 144 public: ModularWritableFile(const std::string & filename,std::unique_ptr<TF_WritableFile> file,const TF_WritableFileOps * ops)145 ModularWritableFile(const std::string& filename, 146 std::unique_ptr<TF_WritableFile> file, 147 const TF_WritableFileOps* ops) 148 : filename_(filename), file_(std::move(file)), ops_(ops) {} 149 ~ModularWritableFile()150 ~ModularWritableFile() override { ops_->cleanup(file_.get()); } 151 152 Status Append(StringPiece data) override; 153 Status Close() override; 154 Status Flush() override; 155 Status Sync() override; 156 Status Name(StringPiece* result) const override; 157 Status Tell(int64_t* position) override; 158 159 private: 160 std::string filename_; 161 std::unique_ptr<TF_WritableFile> file_; 162 const TF_WritableFileOps* ops_; // not owned 163 TF_DISALLOW_COPY_AND_ASSIGN(ModularWritableFile); 164 }; 165 166 class ModularReadOnlyMemoryRegion final : public ReadOnlyMemoryRegion { 167 public: ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region,const TF_ReadOnlyMemoryRegionOps * ops)168 ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region, 169 const TF_ReadOnlyMemoryRegionOps* ops) 170 : region_(std::move(region)), ops_(ops) {} 171 ~ModularReadOnlyMemoryRegion()172 ~ModularReadOnlyMemoryRegion() override { ops_->cleanup(region_.get()); }; 173 data()174 const void* data() override { return ops_->data(region_.get()); } length()175 uint64 length() override { return ops_->length(region_.get()); } 176 177 private: 178 std::unique_ptr<TF_ReadOnlyMemoryRegion> region_; 179 const TF_ReadOnlyMemoryRegionOps* ops_; // not owned 180 TF_DISALLOW_COPY_AND_ASSIGN(ModularReadOnlyMemoryRegion); 181 }; 182 183 // Registers a filesystem plugin so that core TensorFlow can use it. 184 Status RegisterFilesystemPlugin(const std::string& dso_path); 185 186 } // namespace tensorflow 187 188 #endif // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_ 189