1 /*
2 * Copyright 2022 Google LLC
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "fcp/aggregation/testing/testing.h"
18
19 #include <memory>
20 #include <ostream>
21 #include <string>
22 #include <utility>
23
24 #include "fcp/base/platform.h"
25 #include "fcp/tensorflow/status.h"
26 #include "fcp/testing/testing.h"
27 #include "tensorflow/c/checkpoint_reader.h"
28 #include "tensorflow/c/tf_status.h"
29 #include "tensorflow/c/tf_status_helper.h"
30 #include "tensorflow/cc/framework/scope.h"
31 #include "tensorflow/cc/ops/io_ops.h"
32 #include "tensorflow/core/framework/tensor.h"
33 #include "tensorflow/core/public/session.h"
34
35 namespace fcp::aggregation {
36
37 using ::tensorflow::StatusFromTF_Status;
38 using ::tensorflow::TF_StatusPtr;
39 using ::tensorflow::checkpoint::CheckpointReader;
40
operator <<(std::ostream & os,const Tensor & tensor)41 std::ostream& operator<<(std::ostream& os, const Tensor& tensor) {
42 DTYPE_CASES(tensor.dtype(), T,
43 DescribeTensor<T>(&os, tensor.dtype(), tensor.shape(),
44 TensorValuesToVector<T>(tensor)));
45 return os;
46 }
47
CreateStringTfTensor(std::initializer_list<int64_t> dim_sizes,std::initializer_list<string_view> values)48 tf::Tensor CreateStringTfTensor(std::initializer_list<int64_t> dim_sizes,
49 std::initializer_list<string_view> values) {
50 tf::TensorShape shape;
51 EXPECT_TRUE(tf::TensorShape::BuildTensorShape(dim_sizes, &shape).ok());
52 tf::Tensor tensor(tf::DT_STRING, shape);
53 auto* tensor_data_ptr = reinterpret_cast<tf::tstring*>(tensor.data());
54 for (auto value : values) {
55 *tensor_data_ptr++ = value;
56 }
57 return tensor;
58 }
59
CreateTfCheckpoint(tf::Input filename,tf::Input tensor_names,tf::InputList tensors)60 tf::Status CreateTfCheckpoint(tf::Input filename, tf::Input tensor_names,
61 tf::InputList tensors) {
62 tf::Scope scope = tf::Scope::NewRootScope();
63
64 tf::ops::Save save(scope, std::move(filename), std::move(tensor_names),
65 std::move(tensors));
66
67 tf::GraphDef graph;
68 if (auto s = scope.ToGraphDef(&graph); !s.ok()) return s;
69
70 auto session = absl::WrapUnique(tf::NewSession(tf::SessionOptions()));
71 if (auto s = session->Create(graph); !s.ok()) return s;
72 return session->Run({}, {}, {save.operation.node()->name()}, nullptr);
73 }
74
75 absl::StatusOr<absl::flat_hash_map<std::string, std::string>>
SummarizeCheckpoint(const absl::Cord & checkpoint)76 SummarizeCheckpoint(const absl::Cord& checkpoint) {
77 std::string filename = TemporaryTestFile(".ckpt");
78 FCP_RETURN_IF_ERROR(WriteCordToFile(filename, checkpoint));
79
80 TF_StatusPtr tf_status(TF_NewStatus());
81 auto reader = std::make_unique<CheckpointReader>(filename, tf_status.get());
82 FCP_RETURN_IF_ERROR(
83 ConvertFromTensorFlowStatus(StatusFromTF_Status(tf_status.get())));
84
85 absl::flat_hash_map<std::string, std::string> tensors;
86 for (const auto& [name, shape] : reader->GetVariableToShapeMap()) {
87 std::unique_ptr<::tensorflow::Tensor> tensor;
88 reader->GetTensor(name, &tensor, tf_status.get());
89 FCP_RETURN_IF_ERROR(
90 ConvertFromTensorFlowStatus(StatusFromTF_Status(tf_status.get())));
91 tensors[name] = tensor->SummarizeValue(/*max_entries=*/10);
92 }
93 return tensors;
94 }
95 } // namespace fcp::aggregation
96