xref: /aosp_15_r20/external/tensorflow/tensorflow/c/experimental/filesystem/modular_filesystem.h (revision b6fb3261f9314811a0f4371741dbb8839866f948)
1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #ifndef TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
16 #define TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
17 
18 #include <memory>
19 
20 #include "tensorflow/c/experimental/filesystem/filesystem_interface.h"
21 #include "tensorflow/core/platform/file_system.h"
22 
23 /// This file builds classes needed to hold a filesystem implementation in the
24 /// modular world. Once all TensorFlow filesystems are converted to use the
25 /// plugin based approach, this file will replace the one in core/platform and
26 /// the names will lose the `Modular` part. Until that point, the `Modular*`
27 /// classes here are experimental and subject to breaking changes.
28 /// For documentation on these methods, consult `core/platform/filesystem.h`.
29 
30 namespace tensorflow {
31 
32 // TODO(b/143949615): After all filesystems are converted, this file will be
33 // moved to core/platform, and this class can become a singleton and replace the
34 // need for `Env::Default()`. At that time, we might decide to remove the need
35 // for `Env::Default()` altogether, but that's a different project, not in
36 // scope for now. I'm just mentioning this here as that transition will mean
37 // removal of the registration part from `Env` and adding it here instead: we
38 // will need tables to hold for each scheme the function tables that implement
39 // the needed functionality instead of the current `FileSystemRegistry` code in
40 // `core/platform/env.cc`.
41 class ModularFileSystem final : public FileSystem {
42  public:
ModularFileSystem(std::unique_ptr<TF_Filesystem> filesystem,std::unique_ptr<const TF_FilesystemOps> filesystem_ops,std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops,std::unique_ptr<const TF_WritableFileOps> writable_file_ops,std::unique_ptr<const TF_ReadOnlyMemoryRegionOps> read_only_memory_region_ops,std::function<void * (size_t)> plugin_memory_allocate,std::function<void (void *)> plugin_memory_free)43   ModularFileSystem(
44       std::unique_ptr<TF_Filesystem> filesystem,
45       std::unique_ptr<const TF_FilesystemOps> filesystem_ops,
46       std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops,
47       std::unique_ptr<const TF_WritableFileOps> writable_file_ops,
48       std::unique_ptr<const TF_ReadOnlyMemoryRegionOps>
49           read_only_memory_region_ops,
50       std::function<void*(size_t)> plugin_memory_allocate,
51       std::function<void(void*)> plugin_memory_free)
52       : filesystem_(std::move(filesystem)),
53         ops_(std::move(filesystem_ops)),
54         random_access_file_ops_(std::move(random_access_file_ops)),
55         writable_file_ops_(std::move(writable_file_ops)),
56         read_only_memory_region_ops_(std::move(read_only_memory_region_ops)),
57         plugin_memory_allocate_(std::move(plugin_memory_allocate)),
58         plugin_memory_free_(std::move(plugin_memory_free)) {}
59 
~ModularFileSystem()60   ~ModularFileSystem() override { ops_->cleanup(filesystem_.get()); }
61 
62   TF_USE_FILESYSTEM_METHODS_WITH_NO_TRANSACTION_SUPPORT;
63 
64   Status NewRandomAccessFile(
65       const std::string& fname, TransactionToken* token,
66       std::unique_ptr<RandomAccessFile>* result) override;
67   Status NewWritableFile(const std::string& fname, TransactionToken* token,
68                          std::unique_ptr<WritableFile>* result) override;
69   Status NewAppendableFile(const std::string& fname, TransactionToken* token,
70                            std::unique_ptr<WritableFile>* result) override;
71   Status NewReadOnlyMemoryRegionFromFile(
72       const std::string& fname, TransactionToken* token,
73       std::unique_ptr<ReadOnlyMemoryRegion>* result) override;
74   Status FileExists(const std::string& fname, TransactionToken* token) override;
75   bool FilesExist(const std::vector<std::string>& files,
76                   TransactionToken* token,
77                   std::vector<Status>* status) override;
78   Status GetChildren(const std::string& dir, TransactionToken* token,
79                      std::vector<std::string>* result) override;
80   Status GetMatchingPaths(const std::string& pattern, TransactionToken* token,
81                           std::vector<std::string>* results) override;
82   Status DeleteFile(const std::string& fname, TransactionToken* token) override;
83   Status DeleteRecursively(const std::string& dirname, TransactionToken* token,
84                            int64_t* undeleted_files,
85                            int64_t* undeleted_dirs) override;
86   Status DeleteDir(const std::string& dirname,
87                    TransactionToken* token) override;
88   Status RecursivelyCreateDir(const std::string& dirname,
89                               TransactionToken* token) override;
90   Status CreateDir(const std::string& dirname,
91                    TransactionToken* token) override;
92   Status Stat(const std::string& fname, TransactionToken* token,
93               FileStatistics* stat) override;
94   Status IsDirectory(const std::string& fname,
95                      TransactionToken* token) override;
96   Status GetFileSize(const std::string& fname, TransactionToken* token,
97                      uint64* file_size) override;
98   Status RenameFile(const std::string& src, const std::string& target,
99                     TransactionToken* token) override;
100   Status CopyFile(const std::string& src, const std::string& target,
101                   TransactionToken* token) override;
102   std::string TranslateName(const std::string& name) const override;
103   void FlushCaches(TransactionToken* token) override;
104   Status SetOption(const std::string& name,
105                    const std::vector<string>& values) override;
106   Status SetOption(const std::string& name,
107                    const std::vector<int64_t>& values) override;
108   Status SetOption(const std::string& name,
109                    const std::vector<double>& values) override;
110 
111  private:
112   std::unique_ptr<TF_Filesystem> filesystem_;
113   std::unique_ptr<const TF_FilesystemOps> ops_;
114   std::unique_ptr<const TF_RandomAccessFileOps> random_access_file_ops_;
115   std::unique_ptr<const TF_WritableFileOps> writable_file_ops_;
116   std::unique_ptr<const TF_ReadOnlyMemoryRegionOps>
117       read_only_memory_region_ops_;
118   std::function<void*(size_t)> plugin_memory_allocate_;
119   std::function<void(void*)> plugin_memory_free_;
120   TF_DISALLOW_COPY_AND_ASSIGN(ModularFileSystem);
121 };
122 
123 class ModularRandomAccessFile final : public RandomAccessFile {
124  public:
ModularRandomAccessFile(const std::string & filename,std::unique_ptr<TF_RandomAccessFile> file,const TF_RandomAccessFileOps * ops)125   ModularRandomAccessFile(const std::string& filename,
126                           std::unique_ptr<TF_RandomAccessFile> file,
127                           const TF_RandomAccessFileOps* ops)
128       : filename_(filename), file_(std::move(file)), ops_(ops) {}
129 
~ModularRandomAccessFile()130   ~ModularRandomAccessFile() override { ops_->cleanup(file_.get()); }
131 
132   Status Read(uint64 offset, size_t n, StringPiece* result,
133               char* scratch) const override;
134   Status Name(StringPiece* result) const override;
135 
136  private:
137   std::string filename_;
138   std::unique_ptr<TF_RandomAccessFile> file_;
139   const TF_RandomAccessFileOps* ops_;  // not owned
140   TF_DISALLOW_COPY_AND_ASSIGN(ModularRandomAccessFile);
141 };
142 
143 class ModularWritableFile final : public WritableFile {
144  public:
ModularWritableFile(const std::string & filename,std::unique_ptr<TF_WritableFile> file,const TF_WritableFileOps * ops)145   ModularWritableFile(const std::string& filename,
146                       std::unique_ptr<TF_WritableFile> file,
147                       const TF_WritableFileOps* ops)
148       : filename_(filename), file_(std::move(file)), ops_(ops) {}
149 
~ModularWritableFile()150   ~ModularWritableFile() override { ops_->cleanup(file_.get()); }
151 
152   Status Append(StringPiece data) override;
153   Status Close() override;
154   Status Flush() override;
155   Status Sync() override;
156   Status Name(StringPiece* result) const override;
157   Status Tell(int64_t* position) override;
158 
159  private:
160   std::string filename_;
161   std::unique_ptr<TF_WritableFile> file_;
162   const TF_WritableFileOps* ops_;  // not owned
163   TF_DISALLOW_COPY_AND_ASSIGN(ModularWritableFile);
164 };
165 
166 class ModularReadOnlyMemoryRegion final : public ReadOnlyMemoryRegion {
167  public:
ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region,const TF_ReadOnlyMemoryRegionOps * ops)168   ModularReadOnlyMemoryRegion(std::unique_ptr<TF_ReadOnlyMemoryRegion> region,
169                               const TF_ReadOnlyMemoryRegionOps* ops)
170       : region_(std::move(region)), ops_(ops) {}
171 
~ModularReadOnlyMemoryRegion()172   ~ModularReadOnlyMemoryRegion() override { ops_->cleanup(region_.get()); };
173 
data()174   const void* data() override { return ops_->data(region_.get()); }
length()175   uint64 length() override { return ops_->length(region_.get()); }
176 
177  private:
178   std::unique_ptr<TF_ReadOnlyMemoryRegion> region_;
179   const TF_ReadOnlyMemoryRegionOps* ops_;  // not owned
180   TF_DISALLOW_COPY_AND_ASSIGN(ModularReadOnlyMemoryRegion);
181 };
182 
183 // Registers a filesystem plugin so that core TensorFlow can use it.
184 Status RegisterFilesystemPlugin(const std::string& dso_path);
185 
186 }  // namespace tensorflow
187 
188 #endif  // TENSORFLOW_C_EXPERIMENTAL_FILESYSTEM_MODULAR_FILESYSTEM_H_
189