1 //
2 // Copyright (C) 2012 The Android Open Source Project
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16
17 #include "update_engine/payload_generator/delta_diff_generator.h"
18
19 #include <fcntl.h>
20 #include <sys/stat.h>
21 #include <sys/types.h>
22
23 #include <algorithm>
24 #include <memory>
25 #include <string>
26 #include <utility>
27 #include <vector>
28
29 #include <base/logging.h>
30 #include <base/threading/simple_thread.h>
31
32 #include "update_engine/common/utils.h"
33 #include "update_engine/payload_consumer/file_descriptor.h"
34 #include "update_engine/payload_generator/ab_generator.h"
35 #include "update_engine/payload_generator/annotated_operation.h"
36 #include "update_engine/payload_generator/blob_file_writer.h"
37 #include "update_engine/payload_generator/cow_size_estimator.h"
38 #include "update_engine/payload_generator/delta_diff_utils.h"
39 #include "update_engine/payload_generator/full_update_generator.h"
40 #include "update_engine/payload_generator/merge_sequence_generator.h"
41 #include "update_engine/payload_generator/payload_file.h"
42 #include "update_engine/update_metadata.pb.h"
43
44 using std::string;
45 using std::unique_ptr;
46 using std::vector;
47
48 namespace chromeos_update_engine {
49
50 // bytes
51 const size_t kRootFSPartitionSize = static_cast<size_t>(2) * 1024 * 1024 * 1024;
52
53 class PartitionProcessor : public base::DelegateSimpleThread::Delegate {
IsDynamicPartition(const std::string & partition_name)54 bool IsDynamicPartition(const std::string& partition_name) {
55 for (const auto& group :
56 config_.target.dynamic_partition_metadata->groups()) {
57 const auto& names = group.partition_names();
58 if (std::find(names.begin(), names.end(), partition_name) !=
59 names.end()) {
60 return true;
61 }
62 }
63 return false;
64 }
65
66 public:
PartitionProcessor(const PayloadGenerationConfig & config,const PartitionConfig & old_part,const PartitionConfig & new_part,BlobFileWriter * file_writer,std::vector<AnnotatedOperation> * aops,std::vector<CowMergeOperation> * cow_merge_sequence,android::snapshot::CowSizeInfo * cow_info,std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)67 explicit PartitionProcessor(
68 const PayloadGenerationConfig& config,
69 const PartitionConfig& old_part,
70 const PartitionConfig& new_part,
71 BlobFileWriter* file_writer,
72 std::vector<AnnotatedOperation>* aops,
73 std::vector<CowMergeOperation>* cow_merge_sequence,
74 android::snapshot::CowSizeInfo* cow_info,
75 std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy)
76 : config_(config),
77 old_part_(old_part),
78 new_part_(new_part),
79 file_writer_(file_writer),
80 aops_(aops),
81 cow_merge_sequence_(cow_merge_sequence),
82 cow_info_(cow_info),
83 strategy_(std::move(strategy)) {}
84 PartitionProcessor(PartitionProcessor&&) noexcept = default;
85
Run()86 void Run() override {
87 LOG(INFO) << "Started an async task to process partition "
88 << new_part_.name;
89 bool success = strategy_->GenerateOperations(
90 config_, old_part_, new_part_, file_writer_, aops_);
91 if (!success) {
92 // ABORT the entire process, so that developer can look
93 // at recent logs and diagnose what happened
94 LOG(FATAL) << "GenerateOperations(" << old_part_.name << ", "
95 << new_part_.name << ") failed";
96 }
97
98 bool snapshot_enabled =
99 config_.target.dynamic_partition_metadata &&
100 config_.target.dynamic_partition_metadata->snapshot_enabled();
101 if (!snapshot_enabled || !IsDynamicPartition(new_part_.name)) {
102 return;
103 }
104 // Skip cow size estimation if VABC isn't enabled
105 if (!config_.target.dynamic_partition_metadata->vabc_enabled()) {
106 return;
107 }
108 if (!old_part_.path.empty()) {
109 auto generator = MergeSequenceGenerator::Create(*aops_, new_part_.name);
110 if (!generator || !generator->Generate(cow_merge_sequence_)) {
111 LOG(FATAL) << "Failed to generate merge sequence";
112 }
113 }
114
115 LOG(INFO) << "Estimating COW size for partition: " << new_part_.name;
116 // Need the contents of source/target image bytes when doing
117 // dry run.
118 auto target_fd = std::make_unique<EintrSafeFileDescriptor>();
119 target_fd->Open(new_part_.path.c_str(), O_RDONLY);
120
121 google::protobuf::RepeatedPtrField<InstallOperation> operations;
122
123 for (const AnnotatedOperation& aop : *aops_) {
124 *operations.Add() = aop.op;
125 }
126
127 FileDescriptorPtr source_fd = std::make_shared<EintrSafeFileDescriptor>();
128 source_fd->Open(old_part_.path.c_str(), O_RDONLY);
129
130 *cow_info_ = EstimateCowSizeInfo(
131 std::move(source_fd),
132 std::move(target_fd),
133 std::move(operations),
134 {cow_merge_sequence_->begin(), cow_merge_sequence_->end()},
135 config_.block_size,
136 config_.target.dynamic_partition_metadata->vabc_compression_param(),
137 new_part_.size,
138 old_part_.size,
139 config_.enable_vabc_xor,
140 config_.target.dynamic_partition_metadata->cow_version(),
141 config_.target.dynamic_partition_metadata->compression_factor());
142
143 // add a 1% overhead to our estimation
144 cow_info_->cow_size = cow_info_->cow_size * 1.01;
145 if (config_.target.dynamic_partition_metadata->cow_version() >= 3) {
146 cow_info_->op_count_max = std::max(int(cow_info_->op_count_max), 25);
147 }
148 // ops buffer size == 0 for v2 version of cow format
149 LOG(INFO) << "Estimated COW size for partition: " << new_part_.name << " "
150 << cow_info_->cow_size
151 << " ops buffer size: " << cow_info_->op_count_max;
152 }
153
154 private:
155 const PayloadGenerationConfig& config_;
156 const PartitionConfig& old_part_;
157 const PartitionConfig& new_part_;
158 BlobFileWriter* file_writer_;
159 std::vector<AnnotatedOperation>* aops_;
160 std::vector<CowMergeOperation>* cow_merge_sequence_;
161 android::snapshot::CowSizeInfo* cow_info_;
162 std::unique_ptr<chromeos_update_engine::OperationsGenerator> strategy_;
163 DISALLOW_COPY_AND_ASSIGN(PartitionProcessor);
164 };
165
GenerateUpdatePayloadFile(const PayloadGenerationConfig & config,const string & output_path,const string & private_key_path,uint64_t * metadata_size)166 bool GenerateUpdatePayloadFile(const PayloadGenerationConfig& config,
167 const string& output_path,
168 const string& private_key_path,
169 uint64_t* metadata_size) {
170 if (!config.version.Validate()) {
171 LOG(ERROR) << "Unsupported major.minor version: " << config.version.major
172 << "." << config.version.minor;
173 return false;
174 }
175
176 // Create empty payload file object.
177 PayloadFile payload;
178 TEST_AND_RETURN_FALSE(payload.Init(config));
179
180 ScopedTempFile data_file("CrAU_temp_data.XXXXXX", true);
181 {
182 off_t data_file_size = 0;
183 BlobFileWriter blob_file(data_file.fd(), &data_file_size);
184 if (config.is_delta) {
185 TEST_EQ(config.source.partitions.size(), config.target.partitions.size());
186 }
187 PartitionConfig empty_part("");
188 std::vector<std::vector<AnnotatedOperation>> all_aops;
189 all_aops.resize(config.target.partitions.size());
190
191 std::vector<std::vector<CowMergeOperation>> all_merge_sequences;
192 all_merge_sequences.resize(config.target.partitions.size());
193
194 std::vector<android::snapshot::CowSizeInfo> all_cow_info(
195 config.target.partitions.size());
196
197 std::vector<PartitionProcessor> partition_tasks{};
198 auto thread_count = std::min<size_t>(diff_utils::GetMaxThreads(),
199 config.target.partitions.size());
200 if (thread_count > config.max_threads && config.max_threads > 0) {
201 thread_count = config.max_threads;
202 }
203 if (thread_count < 1) {
204 thread_count = 1;
205 }
206 base::DelegateSimpleThreadPool thread_pool{"partition-thread-pool",
207 static_cast<int>(thread_count)};
208 LOG(INFO) << "Using " << thread_count << " threads to process "
209 << config.target.partitions.size() << " partitions";
210 for (size_t i = 0; i < config.target.partitions.size(); i++) {
211 const PartitionConfig& old_part =
212 config.is_delta ? config.source.partitions[i] : empty_part;
213 const PartitionConfig& new_part = config.target.partitions[i];
214 LOG(INFO) << "Partition name: " << new_part.name;
215 LOG(INFO) << "Partition size: " << new_part.size;
216 LOG(INFO) << "Block count: " << new_part.size / config.block_size;
217
218 // Select payload generation strategy based on the config.
219 unique_ptr<OperationsGenerator> strategy;
220 if (!old_part.path.empty()) {
221 // Delta update.
222 LOG(INFO) << "Using generator ABGenerator() for partition "
223 << new_part.name;
224 strategy.reset(new ABGenerator());
225 } else {
226 LOG(INFO) << "Using generator FullUpdateGenerator() for partition "
227 << new_part.name;
228 strategy.reset(new FullUpdateGenerator());
229 }
230
231 // Generate the operations using the strategy we selected above.
232 partition_tasks.push_back(PartitionProcessor(config,
233 old_part,
234 new_part,
235 &blob_file,
236 &all_aops[i],
237 &all_merge_sequences[i],
238 &all_cow_info[i],
239 std::move(strategy)));
240 }
241 thread_pool.Start();
242 for (auto& processor : partition_tasks) {
243 thread_pool.AddWork(&processor);
244 }
245 thread_pool.JoinAll();
246
247 for (size_t i = 0; i < config.target.partitions.size(); i++) {
248 const PartitionConfig& old_part =
249 config.is_delta ? config.source.partitions[i] : empty_part;
250 const PartitionConfig& new_part = config.target.partitions[i];
251 TEST_AND_RETURN_FALSE(
252 payload.AddPartition(old_part,
253 new_part,
254 std::move(all_aops[i]),
255 std::move(all_merge_sequences[i]),
256 all_cow_info[i]));
257 }
258 }
259 data_file.CloseFd();
260
261 LOG(INFO) << "Writing payload file...";
262 // Write payload file to disk.
263 TEST_AND_RETURN_FALSE(payload.WritePayload(
264 output_path, data_file.path(), private_key_path, metadata_size));
265
266 LOG(INFO) << "All done. Successfully created delta file with "
267 << "metadata size = " << *metadata_size;
268 return true;
269 }
270
271 }; // namespace chromeos_update_engine
272