1 // Copyright 2023 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 use std::collections::HashMap;
6 use std::env;
7 use std::process::Command;
8 use std::process::Stdio;
9 use std::thread;
10 use std::time::Duration;
11
12 use anyhow::bail;
13 use anyhow::Context;
14 use anyhow::Result;
15 use rayon::prelude::*;
16
17 /// The target device running crosvm to collect information from.
18 struct Target {
19 /// SSH host name.
20 host: String,
21 }
22
23 impl Target {
do_command(&self, command: Vec<&str>) -> Result<String>24 fn do_command(&self, command: Vec<&str>) -> Result<String> {
25 let child = Command::new("ssh")
26 .arg(&self.host)
27 .args(&command)
28 .stdout(Stdio::piped())
29 .stderr(Stdio::piped())
30 .spawn()
31 .context("failed to execute process")?;
32 let output = child
33 .wait_with_output()
34 .context("failed to wait on child")?;
35 if !output.status.success() {
36 bail!(format!("{:?}: output status: {}", command, output.status));
37 }
38 Ok(String::from_utf8(output.stdout).context("Failed to convert command output to utf8")?)
39 }
40
get_file(&self, filename: &str) -> Result<String>41 fn get_file(&self, filename: &str) -> Result<String> {
42 Ok(self.do_command(vec!["cat", filename])?)
43 }
44
do_fincore(&self, filenames: &Vec<String>) -> Result<Vec<(u64, u64)>>45 fn do_fincore(&self, filenames: &Vec<String>) -> Result<Vec<(u64, u64)>> {
46 let mut command = vec!["fincore", "--raw", "--bytes"];
47 command.extend(filenames.iter().map(|x| &**x));
48 parse_fincore(&self.do_command(command)?)
49 }
50 }
51
parse_fincore(text: &str) -> Result<Vec<(u64, u64)>>52 fn parse_fincore(text: &str) -> Result<Vec<(u64, u64)>> {
53 let mut result = vec![];
54 for line in text.lines().skip(1) {
55 // res(bytes) pages size filename.
56 let mut words = line.split(" ");
57 let resident =
58 str::parse::<u64>(words.next().context("res")?).context("number from fincore")?;
59 let _pages = words.next().context("pages")?;
60 let size =
61 str::parse::<u64>(words.next().context("size")?).context("number from fincore")?;
62 result.push((resident, size));
63 }
64 Ok(result)
65 }
66
67 // Extract only lines with a number as the second parameter, from /proc/pid/status
parse_status(text: &str) -> Result<std::collections::HashMap<&str, u32>>68 fn parse_status(text: &str) -> Result<std::collections::HashMap<&str, u32>> {
69 let key_value_iter = text
70 .lines()
71 .skip(1)
72 .filter_map(|line| {
73 let mut split = line.split_whitespace();
74 let key = split.next().expect("key");
75 if let Ok(value) = str::parse::<u32>(split.next().expect("number")) {
76 Some((key, value))
77 } else {
78 None
79 }
80 })
81 .collect::<Vec<_>>();
82 let key_value_map: HashMap<_, _> = HashMap::from_iter(key_value_iter);
83 Ok(key_value_map)
84 }
85
parse_smaps(smaps_rollup_text: &str) -> std::collections::HashMap<&str, u32>86 fn parse_smaps(smaps_rollup_text: &str) -> std::collections::HashMap<&str, u32> {
87 let key_value_iter = smaps_rollup_text.lines().skip(1).map(|x| {
88 let mut split = x.split_whitespace();
89 let key = split.next().expect("key");
90 let value = str::parse::<u32>(split.next().unwrap()).expect("kB") * 1024;
91 let kb = split.next().unwrap();
92 assert!(kb == "kB");
93 (key, value)
94 });
95 let key_value_map: HashMap<_, _> = HashMap::from_iter(key_value_iter);
96 key_value_map
97 }
98
99 struct BlockFds<'a> {
100 // fd number.
101 fd: u32,
102 // The file path that the fd points to.
103 path: &'a str,
104 }
105
find_block_fds(proc_fd: &str) -> Vec<BlockFds>106 fn find_block_fds(proc_fd: &str) -> Vec<BlockFds> {
107 proc_fd
108 .lines()
109 .skip(1)
110 .filter_map(|line| {
111 let items: Vec<_> = line.split_whitespace().collect();
112 assert_eq!(items[9], "->");
113 let path = items[10];
114 if path.contains("/memfd:")
115 || path.contains("/dev/kvm")
116 || path.contains("/dev/null")
117 || path.contains("/dev/net/")
118 || path.contains("/dev/dri/")
119 || path.contains("/sys/fs")
120 || path == "/"
121 {
122 None
123 } else if path.contains("/") {
124 Some(BlockFds {
125 fd: items[8].parse::<u32>().unwrap(),
126 path,
127 })
128 } else {
129 None
130 }
131 })
132 .collect()
133 }
134
parse_fd_blocks(target: &Target, who: &str, pid: u32) -> Result<String>135 fn parse_fd_blocks(target: &Target, who: &str, pid: u32) -> Result<String> {
136 let lines = target
137 .do_command(vec!["ls", "-l", &format!("/proc/{}/fd/", pid)])
138 .context("ls -l for proc/fd")?;
139 let block_fds = find_block_fds(&lines);
140 let message = block_fds.par_iter().map(|block_fd| {
141 let fdinfo = target.get_file(&format!("/proc/{}/fdinfo/{}", pid, block_fd.fd)).expect("/proc/fdinfo");
142 let flags = u32::from_str_radix(parse_proc_fdinfo_flags(&fdinfo), 8).expect("octal");
143 let fincore = target.do_fincore(&vec![block_fd.path.to_string()]).unwrap();
144 assert_eq!(fincore.len(), 1);
145
146 format!(
147 "{} {} {} flags: {:o} o_direct on x86_64 {}, o_direct on arm {} page cache: {} MB / {} MB",
148 who,
149 block_fd.path,
150 block_fd.fd,
151 flags,
152 (flags & 0o40000) != 0,
153 (flags & 0o200000) != 0,
154 fincore[0].0 >> 20,
155 fincore[0].1 >> 20,
156 )
157 }).collect::<Vec<_>>().join("\n");
158 Ok(message)
159 }
160
parse_proc_fdinfo_flags(proc_fdinfo: &str) -> &str161 fn parse_proc_fdinfo_flags(proc_fdinfo: &str) -> &str {
162 let lines: HashMap<_, _> = proc_fdinfo
163 .lines()
164 .map(|line| {
165 let mut words = line.split(":");
166 (words.next().unwrap(), words.next().unwrap().trim())
167 })
168 .collect();
169 lines["flags"]
170 }
171
parse_virtio_fs(target: &Target, pid: u32) -> Result<String>172 fn parse_virtio_fs(target: &Target, pid: u32) -> Result<String> {
173 let lines = target.do_command(vec!["ls", "-1", &format!("/proc/{}/task/", pid)])?;
174 let task_pids: Vec<_> = lines.lines().map(|x| x.parse::<u32>().unwrap()).collect();
175 let pid_name_pairs: Vec<_> = task_pids
176 .par_iter()
177 .map(|task_pid| {
178 let comm = target
179 .get_file(&format!("/proc/{}/comm", task_pid))
180 .unwrap();
181 let comm = comm.trim();
182 (task_pid, comm.to_string())
183 })
184 .collect();
185 let message = pid_name_pairs
186 .iter()
187 .map(|(task_pid, comm)| format!("{}:{}", task_pid, comm))
188 .collect::<Vec<_>>()
189 .join(" ");
190 Ok(message)
191 }
192
main() -> Result<()>193 fn main() -> Result<()> {
194 // Use a little less than 10 threads globally. 10 sessions is the limit on
195 // sshd connection by default as it is on the chromebook.
196 rayon::ThreadPoolBuilder::new()
197 .num_threads(8)
198 .build_global()
199 .unwrap();
200
201 let host = env::args().nth(1).unwrap();
202 let target = Target { host: host.clone() };
203 while target.do_command(vec!["uname", "-a"]).is_err() {
204 println!("Retrying {}", host);
205 thread::sleep(Duration::from_millis(1000));
206 }
207 let crosvm_pid =
208 str::parse::<u32>(&target.do_command(vec!["pgrep", "crosvm"]).unwrap().trim()).unwrap();
209 let crosvm_cmdline = target.get_file(&format!("/proc/{}/cmdline", crosvm_pid))?;
210 let commandline_flags: Vec<_> = crosvm_cmdline.split("\0").collect();
211
212 let mut shared_dir_params = vec![];
213 let mut disk_params = vec![];
214 let mut socket = "";
215 for (i, line) in commandline_flags.iter().enumerate() {
216 match *line {
217 "--shared-dir" => shared_dir_params.push(commandline_flags[i + 1]),
218 "--block" => disk_params.push(commandline_flags[i + 1]),
219 "--socket" => socket = commandline_flags[i + 1],
220 _ => {
221 // Skip other flags.
222 }
223 }
224 }
225 println!("{:?}", shared_dir_params);
226 println!("{:?}", disk_params);
227
228 // Parsed command line for paths to virtio disk blocks. Concierge gives links to /proc/self/fd,
229 // translate them to actual end paths after resolving symlinks.
230 let disk_blocks: Vec<_> = disk_params
231 .par_iter()
232 .map(|disk| {
233 let block_path_in_proc = disk.split(",").nth(0).unwrap();
234 // this would be like /proc/self/fd/26
235 assert!(block_path_in_proc.starts_with("/proc/self/fd/"));
236 let fd_id =
237 str::parse::<u32>(block_path_in_proc.split("/").nth(4).unwrap().trim()).unwrap();
238 let disk_block = target
239 .do_command(vec![
240 "readlink",
241 &format!("/proc/{}/fd/{}", crosvm_pid, fd_id),
242 ])
243 .unwrap();
244 disk_block.trim().to_string()
245 })
246 .collect();
247 println!("{:?}", disk_blocks);
248
249 // Get fincore stats.
250 for (i, (res, size)) in target.do_fincore(&disk_blocks).unwrap().iter().enumerate() {
251 println!(
252 "fincore {}: {} MB / {} MB",
253 disk_blocks[i],
254 res >> 20,
255 size >> 20
256 );
257 }
258
259 // Look at fds of crosvm map
260 parse_fd_blocks(&target, "crosvm", crosvm_pid)?;
261
262 // find children of the process
263 let crosvm_child_pids: Vec<_> = target
264 .get_file(&format!(
265 "/proc/{}/task/{}/children",
266 crosvm_pid, crosvm_pid
267 ))?
268 .trim()
269 .split(" ")
270 .map(|x| str::parse::<u32>(x).expect("pid"))
271 .collect();
272
273 // Scanning for crosvm child processes
274 crosvm_child_pids.par_iter().for_each(|child_pid| {
275 let task_name = target
276 .get_file(&format!("/proc/{}/comm", child_pid))
277 .unwrap()
278 .trim()
279 .to_string();
280 // task/*/comm contains thread names which are useful to tell the device
281 // type. smaps_rollup would be useful, smaps too.
282 // values are in kB.
283 let smaps_rollup = target
284 .get_file(&format!("/proc/{}/smaps_rollup", child_pid))
285 .unwrap();
286 let parsed_smaps = parse_smaps(&smaps_rollup);
287 let dirty = parsed_smaps["Private_Dirty:"];
288 let rss = parsed_smaps["Rss:"];
289 let status_text = target
290 .get_file(&format!("/proc/{}/status", child_pid))
291 .unwrap();
292 let vmpte_kb = parse_status(&status_text).unwrap()["VmPTE:"];
293 let message = match task_name.as_str() {
294 "pcivirtio-block" => parse_fd_blocks(&target, "virtio-block", *child_pid),
295 "pcivirtio-fs" => parse_virtio_fs(&target, *child_pid),
296 _ => Ok("".to_string()),
297 }
298 .unwrap();
299
300 // output in MBs.
301 println!(
302 "{} {} private_dirty: {} MB rss: {} MB VmPTE: {} KiB\n {}",
303 task_name,
304 child_pid,
305 dirty >> 20,
306 rss >> 20,
307 vmpte_kb,
308 message
309 );
310 });
311
312 let balloon_stat_json = target.do_command(vec!["crosvm", "balloon_stats", socket])?;
313 println!("{}", balloon_stat_json);
314
315 Ok(())
316 }
317
318 #[cfg(test)]
319 mod tests {
320 use super::*;
321
322 #[test]
parse_smaps_basic()323 fn parse_smaps_basic() {
324 let smaps_rollup_text =
325 "55c9706ac000-7fff1d38e000 ---p 00000000 00:00 0 [rollup]
326 Rss: 5940 kB
327 Pss: 580 kB
328 Pss_Anon: 367 kB
329 Pss_File: 213 kB
330 Pss_Shmem: 0 kB
331 Shared_Clean: 3760 kB
332 Shared_Dirty: 1816 kB
333 Private_Clean: 64 kB
334 Private_Dirty: 300 kB
335 Referenced: 4244 kB
336 Anonymous: 2116 kB
337 LazyFree: 0 kB
338 AnonHugePages: 0 kB
339 ShmemPmdMapped: 0 kB
340 FilePmdMapped: 0 kB
341 Shared_Hugetlb: 0 kB
342 Private_Hugetlb: 0 kB
343 Swap: 0 kB
344 SwapPss: 0 kB
345 Locked: 0 kB
346 ";
347 let key_value_map = parse_smaps(smaps_rollup_text);
348 assert_eq!(key_value_map["Private_Dirty:"], 300 * 1024);
349 }
350
351 #[test]
parse_status_basic() -> Result<()>352 fn parse_status_basic() -> Result<()> {
353 let status_text = "Name: pcivirtio-block
354 Umask: 0002
355 State: S (sleeping)
356 Tgid: 22698
357 Ngid: 0
358 Pid: 22698
359 PPid: 22560
360 TracerPid: 0
361 Uid: 299 299 299 299
362 Gid: 299 299 299 299
363 FDSize: 512
364 Groups: 27 299 333 400 413 418 600 601 603 20128 20136 20162
365 NStgid: 22698 39 1
366 NSpid: 22698 39 1
367 NSpgid: 22560 12 0
368 NSsid: 22142 0 0
369 VmPeak: 15278960 kB
370 VmSize: 15213424 kB
371 VmLck: 0 kB
372 VmPin: 0 kB
373 VmHWM: 176796 kB
374 VmRSS: 176796 kB
375 RssAnon: 2320 kB
376 RssFile: 6540 kB
377 RssShmem: 167936 kB
378 VmData: 3332 kB
379 VmStk: 136 kB
380 VmExe: 10628 kB
381 VmLib: 15436 kB
382 VmPTE: 132 kB
383 VmSwap: 0 kB
384 CoreDumping: 0
385 THP_enabled: 1
386 Threads: 2
387 SigQ: 0/63125
388 SigPnd: 0000000000000000
389 ShdPnd: 0000000000000000
390 SigBlk: 0000000000010000
391 SigIgn: 0000000000001000
392 SigCgt: 0000000100000440
393 CapInh: 0000000000000000
394 CapPrm: 0000000000000000
395 CapEff: 0000000000000000
396 CapBnd: 0000000000000000
397 CapAmb: 0000000000000000
398 NoNewPrivs: 1
399 Seccomp: 2
400 Seccomp_filters: 3
401 Speculation_Store_Bypass: thread force mitigated
402 SpeculationIndirectBranch: conditional force disabled
403 Cpus_allowed: fff
404 Cpus_allowed_list: 0-11
405 Mems_allowed: 1
406 Mems_allowed_list: 0
407 voluntary_ctxt_switches: 533
408 nonvoluntary_ctxt_switches: 3
409 ";
410 let status = parse_status(status_text)?;
411 assert_eq!(status["VmPTE:"], 132);
412 Ok(())
413 }
414
415 #[test]
fincore_test() -> Result<()>416 fn fincore_test() -> Result<()> {
417 let fincore_output = "RES PAGES SIZE FILE\n474968064 115959 667250688 /opt/google/vms/android/system.raw.img\n10452992 2552 140140544 /opt/google/vms/android/vendor.raw.img\n0 0 0 /dev/null\n0 0 0 /dev/null\n215543808 52623 11468791808 /run/daemon-store/crosvm/8b3488f8d78a9827054a417ae7a4b9bb62586267/YXJjdm0=.img\n";
418 parse_fincore(fincore_output)?;
419 Ok(())
420 }
421
422 #[test]
proc_fd_test()423 fn proc_fd_test() {
424 let proc_fd = "total 0
425 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 54 -> 'anon_inode:[eventfd]'
426 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 55 -> 'anon_inode:[eventfd]'
427 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 56 -> 'anon_inode:[eventfd]'
428 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 6 -> 'anon_inode:[eventfd]'
429 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 7 -> '/memfd:crosvm_guest (deleted)'
430 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 8 -> 'anon_inode:[eventfd]'
431 lr-x------. 1 crosvm crosvm 64 Jul 9 09:20 85 -> /opt/google/vms/android/system.raw.img
432 lrwx------. 1 crosvm crosvm 64 Jul 19 09:20 9 -> 'anon_inode:[eventfd]'
433 ";
434
435 let v = find_block_fds(proc_fd);
436 assert_eq!(v.len(), 1);
437 assert_eq!(v[0].fd, 85);
438 assert_eq!(v[0].path, "/opt/google/vms/android/system.raw.img");
439 }
440
441 #[test]
test_fdinfo()442 fn test_fdinfo() {
443 let proc_fdinfo = "pos: 0
444 flags: 0100002
445 mnt_id: 25
446 ino: 18
447 ";
448
449 assert_eq!(parse_proc_fdinfo_flags(proc_fdinfo), "0100002");
450 }
451 }
452