1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "host/libs/vm_manager/qemu_manager.h"
18 
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23 #include <sys/un.h>
24 #include <sys/wait.h>
25 #include <unistd.h>
26 
27 #include <cstdlib>
28 #include <iomanip>
29 #include <string>
30 #include <unordered_map>
31 #include <utility>
32 #include <vector>
33 
34 #include <android-base/strings.h>
35 #include <android-base/logging.h>
36 #include <vulkan/vulkan.h>
37 
38 #include "common/libs/utils/files.h"
39 #include "common/libs/utils/result.h"
40 #include "common/libs/utils/subprocess.h"
41 #include "host/libs/config/command_source.h"
42 #include "host/libs/config/cuttlefish_config.h"
43 #include "host/libs/vm_manager/vhost_user.h"
44 
45 namespace cuttlefish {
46 namespace vm_manager {
47 namespace {
48 
GetMonitorPath(const CuttlefishConfig & config)49 std::string GetMonitorPath(const CuttlefishConfig& config) {
50   return config.ForDefaultInstance().PerInstanceInternalUdsPath(
51       "qemu_monitor.sock");
52 }
53 
Stop()54 StopperResult Stop() {
55   auto config = CuttlefishConfig::Get();
56   auto monitor_path = GetMonitorPath(*config);
57   auto monitor_sock = SharedFD::SocketLocalClient(
58       monitor_path.c_str(), false, SOCK_STREAM);
59 
60   if (!monitor_sock->IsOpen()) {
61     LOG(ERROR) << "The connection to qemu is closed, is it still running?";
62     return StopperResult::kStopFailure;
63   }
64   char msg[] = "{\"execute\":\"qmp_capabilities\"}{\"execute\":\"quit\"}";
65   ssize_t len = sizeof(msg) - 1;
66   while (len > 0) {
67     int tmp = monitor_sock->Write(msg, len);
68     if (tmp < 0) {
69       LOG(ERROR) << "Error writing to socket: " << monitor_sock->StrError();
70       return StopperResult::kStopFailure;
71     }
72     len -= tmp;
73   }
74   // Log the reply
75   char buff[1000];
76   while ((len = monitor_sock->Read(buff, sizeof(buff) - 1)) > 0) {
77     buff[len] = '\0';
78     LOG(INFO) << "From qemu monitor: " << buff;
79   }
80 
81   return StopperResult::kStopSuccess;
82 }
83 
GetQemuVersion(const std::string & qemu_binary)84 Result<std::pair<int, int>> GetQemuVersion(const std::string& qemu_binary) {
85   Command qemu_version_cmd(qemu_binary);
86   qemu_version_cmd.AddParameter("-version");
87 
88   std::string qemu_version_input, qemu_version_output, qemu_version_error;
89   cuttlefish::SubprocessOptions options;
90   options.Verbose(false);
91   int qemu_version_ret = cuttlefish::RunWithManagedStdio(
92       std::move(qemu_version_cmd), &qemu_version_input, &qemu_version_output,
93       &qemu_version_error, std::move(options));
94   CF_EXPECT(qemu_version_ret == 0,
95             qemu_binary << " -version returned unexpected response "
96                         << qemu_version_output << ". Stderr was "
97                         << qemu_version_error);
98 
99   // Snip around the extra text we don't care about
100   qemu_version_output.erase(0, std::string("QEMU emulator version ").length());
101   auto space_pos = qemu_version_output.find(" ", 0);
102   if (space_pos != std::string::npos) {
103     qemu_version_output.resize(space_pos);
104   }
105 
106   auto qemu_version_bits = android::base::Split(qemu_version_output, ".");
107   return {{std::stoi(qemu_version_bits[0]), std::stoi(qemu_version_bits[1])}};
108 }
109 
110 }  // namespace
111 
QemuManager(Arch arch)112 QemuManager::QemuManager(Arch arch) : arch_(arch) {}
113 
IsSupported()114 bool QemuManager::IsSupported() {
115   return HostSupportsQemuCli();
116 }
117 
118 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)119 QemuManager::ConfigureGraphics(
120     const CuttlefishConfig::InstanceSpecific& instance) {
121   // Override the default HAL search paths in all cases. We do this because
122   // the HAL search path allows for fallbacks, and fallbacks in conjunction
123   // with properties lead to non-deterministic behavior while loading the
124   // HALs.
125 
126   std::unordered_map<std::string, std::string> bootconfig_args;
127   auto gpu_mode = instance.gpu_mode();
128   if (gpu_mode == kGpuModeGuestSwiftshader) {
129     bootconfig_args = {
130         {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_2)},
131         {"androidboot.hardware.gralloc", "minigbm"},
132         {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
133         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
134         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
135          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
136         {"androidboot.hardware.egl", "angle"},
137         {"androidboot.hardware.vulkan", "pastel"},
138         // OpenGL ES 3.1
139         {"androidboot.opengles.version", "196609"},
140     };
141   } else if (gpu_mode == kGpuModeDrmVirgl) {
142     bootconfig_args = {
143         {"androidboot.cpuvulkan.version", "0"},
144         {"androidboot.hardware.gralloc", "minigbm"},
145         {"androidboot.hardware.hwcomposer", "ranchu"},
146         {"androidboot.hardware.hwcomposer.mode", "client"},
147         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
148         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
149          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
150         {"androidboot.hardware.egl", "mesa"},
151         // No "hardware" Vulkan support, yet
152         // OpenGL ES 3.0
153         {"androidboot.opengles.version", "196608"},
154     };
155   } else if (gpu_mode == kGpuModeGfxstream ||
156              gpu_mode == kGpuModeGfxstreamGuestAngle ||
157              gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
158     const bool uses_angle =
159         gpu_mode == kGpuModeGfxstreamGuestAngle ||
160         gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader;
161     const std::string gles_impl = uses_angle ? "angle" : "emulation";
162     const std::string gltransport =
163         (instance.guest_android_version() == "11.0.0") ? "virtio-gpu-pipe"
164                                                        : "virtio-gpu-asg";
165     bootconfig_args = {
166         {"androidboot.cpuvulkan.version", "0"},
167         {"androidboot.hardware.gralloc", "minigbm"},
168         {"androidboot.hardware.hwcomposer", instance.hwcomposer()},
169         {"androidboot.hardware.hwcomposer.display_finder_mode", "drm"},
170         {"androidboot.hardware.hwcomposer.display_framebuffer_format",
171          instance.guest_uses_bgra_framebuffers() ? "bgra" : "rgba"},
172         {"androidboot.hardware.egl", gles_impl},
173         {"androidboot.hardware.vulkan", "ranchu"},
174         {"androidboot.hardware.gltransport", gltransport},
175         {"androidboot.opengles.version", "196609"},  // OpenGL ES 3.1
176     };
177   } else if (instance.gpu_mode() == kGpuModeNone) {
178     return {};
179   } else {
180     return CF_ERR("Unhandled GPU mode: " << instance.gpu_mode());
181   }
182 
183   if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
184     bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
185         instance.gpu_angle_feature_overrides_enabled();
186   }
187   if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
188     bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
189         instance.gpu_angle_feature_overrides_disabled();
190   }
191 
192   return bootconfig_args;
193 }
194 
195 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(const CuttlefishConfig::InstanceSpecific & instance)196 QemuManager::ConfigureBootDevices(
197     const CuttlefishConfig::InstanceSpecific& instance) {
198   const int num_disks = instance.virtual_disk_paths().size();
199   const int num_gpu = instance.hwcomposer() != kHwComposerNone;
200   switch (arch_) {
201     case Arch::Arm:
202       return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
203     case Arch::Arm64:
204 #ifdef __APPLE__
205       return {{{"androidboot.boot_devices", "3f000000.pcie"}}};
206 #else
207       return {{{"androidboot.boot_devices", "4010000000.pcie"}}};
208 #endif
209     case Arch::RiscV64:
210       return {{{"androidboot.boot_devices", "soc/30000000.pci"}}};
211     case Arch::X86:
212     case Arch::X86_64: {
213       // QEMU has additional PCI devices for an ISA bridge and PIIX4
214       // virtio_gpu precedes the first console or disk
215       // TODO(schuffelen): Simplify this logic when crosvm uses multiport
216       int pci_offset = 3 + num_gpu - VmManager::kDefaultNumHvcs;
217       return ConfigureMultipleBootDevices("pci0000:00/0000:00:", pci_offset,
218                                           num_disks);
219     }
220   }
221 }
222 
StartCommands(const CuttlefishConfig & config,std::vector<VmmDependencyCommand * > & dependency_commands)223 Result<std::vector<MonitorCommand>> QemuManager::StartCommands(
224     const CuttlefishConfig& config,
225     std::vector<VmmDependencyCommand*>& dependency_commands) {
226   std::vector<MonitorCommand> commands;
227   auto instance = config.ForDefaultInstance();
228   std::string qemu_binary = instance.qemu_binary_dir();
229   switch (arch_) {
230     case Arch::Arm:
231       qemu_binary += "/qemu-system-arm";
232       break;
233     case Arch::Arm64:
234       qemu_binary += "/qemu-system-aarch64";
235       break;
236     case Arch::RiscV64:
237       qemu_binary += "/qemu-system-riscv64";
238       break;
239     case Arch::X86:
240       qemu_binary += "/qemu-system-i386";
241       break;
242     case Arch::X86_64:
243       qemu_binary += "/qemu-system-x86_64";
244       break;
245   }
246 
247   auto qemu_version = CF_EXPECT(GetQemuVersion(qemu_binary));
248   Command qemu_cmd(qemu_binary, KillSubprocessFallback(Stop));
249 
250   qemu_cmd.AddPrerequisite([&dependency_commands]() -> Result<void> {
251     for (auto dependencyCommand : dependency_commands) {
252       CF_EXPECT(dependencyCommand->WaitForAvailability());
253     }
254 
255     return {};
256   });
257 
258   int hvc_num = 0;
259   int serial_num = 0;
260   auto add_hvc_sink = [&qemu_cmd, &hvc_num]() {
261     qemu_cmd.AddParameter("-chardev");
262     qemu_cmd.AddParameter("null,id=hvc", hvc_num);
263     qemu_cmd.AddParameter("-device");
264     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
265                           hvc_num);
266     hvc_num++;
267   };
268   auto add_serial_sink = [&qemu_cmd, &serial_num]() {
269     qemu_cmd.AddParameter("-chardev");
270     qemu_cmd.AddParameter("null,id=serial", serial_num);
271     qemu_cmd.AddParameter("-serial");
272     qemu_cmd.AddParameter("chardev:serial", serial_num);
273     serial_num++;
274   };
275   auto add_serial_console_ro = [&qemu_cmd,
276                                 &serial_num](const std::string& output) {
277     qemu_cmd.AddParameter("-chardev");
278     qemu_cmd.AddParameter("file,id=serial", serial_num, ",path=", output,
279                           ",append=on");
280     qemu_cmd.AddParameter("-serial");
281     qemu_cmd.AddParameter("chardev:serial", serial_num);
282     serial_num++;
283   };
284   auto add_serial_console = [&qemu_cmd,
285                              &serial_num](const std::string& prefix) {
286     qemu_cmd.AddParameter("-chardev");
287     qemu_cmd.AddParameter("pipe,id=serial", serial_num, ",path=", prefix);
288     qemu_cmd.AddParameter("-serial");
289     qemu_cmd.AddParameter("chardev:serial", serial_num);
290     serial_num++;
291   };
292   auto add_hvc_ro = [&qemu_cmd, &hvc_num](const std::string& output) {
293     qemu_cmd.AddParameter("-chardev");
294     qemu_cmd.AddParameter("file,id=hvc", hvc_num, ",path=", output,
295                           ",append=on");
296     qemu_cmd.AddParameter("-device");
297     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
298                           hvc_num);
299     hvc_num++;
300   };
301   auto add_hvc = [&qemu_cmd, &hvc_num](const std::string& prefix) {
302     qemu_cmd.AddParameter("-chardev");
303     qemu_cmd.AddParameter("pipe,id=hvc", hvc_num, ",path=", prefix);
304     qemu_cmd.AddParameter("-device");
305     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
306                           hvc_num);
307     hvc_num++;
308   };
309   auto add_hvc_serial = [&qemu_cmd, &hvc_num](const std::string& prefix) {
310     qemu_cmd.AddParameter("-chardev");
311     qemu_cmd.AddParameter("serial,id=hvc", hvc_num, ",path=", prefix);
312     qemu_cmd.AddParameter("-device");
313     qemu_cmd.AddParameter("virtconsole,bus=virtio-serial.0,chardev=hvc",
314                           hvc_num);
315     hvc_num++;
316   };
317 
318   bool is_arm = arch_ == Arch::Arm || arch_ == Arch::Arm64;
319   bool is_x86 = arch_ == Arch::X86 || arch_ == Arch::X86_64;
320   bool is_riscv64 = arch_ == Arch::RiscV64;
321 
322   auto access_kregistry_size_bytes = 0;
323   if (FileExists(instance.access_kregistry_path())) {
324     access_kregistry_size_bytes = FileSize(instance.access_kregistry_path());
325     CF_EXPECT((access_kregistry_size_bytes & (1024 * 1024 - 1)) == 0,
326               instance.access_kregistry_path()
327                   << " file size (" << access_kregistry_size_bytes
328                   << ") not a multiple of 1MB");
329   }
330 
331   auto hwcomposer_pmem_size_bytes = 0;
332   if (instance.hwcomposer() != kHwComposerNone) {
333     if (FileExists(instance.hwcomposer_pmem_path())) {
334       hwcomposer_pmem_size_bytes = FileSize(instance.hwcomposer_pmem_path());
335       CF_EXPECT((hwcomposer_pmem_size_bytes & (1024 * 1024 - 1)) == 0,
336                 instance.hwcomposer_pmem_path()
337                     << " file size (" << hwcomposer_pmem_size_bytes
338                     << ") not a multiple of 1MB");
339     }
340   }
341 
342   auto pstore_size_bytes = 0;
343   if (FileExists(instance.pstore_path())) {
344     pstore_size_bytes = FileSize(instance.pstore_path());
345     CF_EXPECT((pstore_size_bytes & (1024 * 1024 - 1)) == 0,
346               instance.pstore_path() << " file size (" << pstore_size_bytes
347                                      << ") not a multiple of 1MB");
348   }
349 
350   qemu_cmd.AddParameter("-name");
351   qemu_cmd.AddParameter("guest=", instance.instance_name(), ",debug-threads=on");
352 
353   qemu_cmd.AddParameter("-machine");
354   std::string machine = is_x86 ? "pc,nvdimm=on" : "virt";
355   if (IsHostCompatible(arch_)) {
356 #ifdef __linux__
357     machine += ",accel=kvm";
358 #elif defined(__APPLE__)
359     machine += ",accel=hvf";
360 #else
361 #error "Unknown OS"
362 #endif
363     if (is_arm) {
364       machine += ",gic-version=3";
365     }
366   } else if (is_arm) {
367     // QEMU doesn't support GICv3 with TCG yet
368     machine += ",gic-version=2";
369     CF_EXPECT(instance.cpus() <= 8, "CPUs must be no more than 8 with GICv2");
370   }
371   if (instance.mte()) {
372     machine += ",mte=on";
373   }
374   qemu_cmd.AddParameter(machine, ",usb=off,dump-guest-core=off");
375 
376   qemu_cmd.AddParameter("-m");
377   auto maxmem = instance.memory_mb() +
378                 (access_kregistry_size_bytes / 1024 / 1024) +
379                 (hwcomposer_pmem_size_bytes / 1024 / 1024) +
380                 (is_x86 ? pstore_size_bytes / 1024 / 1024 : 0);
381   auto slots = is_x86 ? ",slots=2" : "";
382   qemu_cmd.AddParameter("size=", instance.memory_mb(), "M",
383                         ",maxmem=", maxmem, "M", slots);
384 
385   qemu_cmd.AddParameter("-overcommit");
386   qemu_cmd.AddParameter("mem-lock=off");
387 
388   // Assume SMT is always 2 threads per core, which is how most hardware
389   // today is configured, and the way crosvm does it
390   qemu_cmd.AddParameter("-smp");
391   if (instance.smt()) {
392     CF_EXPECT(instance.cpus() % 2 == 0,
393               "CPUs must be a multiple of 2 in SMT mode");
394     qemu_cmd.AddParameter(instance.cpus(), ",cores=",
395                           instance.cpus() / 2, ",threads=2");
396   } else {
397     qemu_cmd.AddParameter(instance.cpus(), ",cores=",
398                           instance.cpus(), ",threads=1");
399   }
400 
401   qemu_cmd.AddParameter("-uuid");
402   qemu_cmd.AddParameter(instance.uuid());
403 
404   qemu_cmd.AddParameter("-no-user-config");
405   qemu_cmd.AddParameter("-nodefaults");
406   qemu_cmd.AddParameter("-no-shutdown");
407 
408   qemu_cmd.AddParameter("-rtc");
409   qemu_cmd.AddParameter("base=utc");
410 
411   qemu_cmd.AddParameter("-boot");
412   qemu_cmd.AddParameter("strict=on");
413 
414   qemu_cmd.AddParameter("-chardev");
415   qemu_cmd.AddParameter("socket,id=charmonitor,path=", GetMonitorPath(config),
416                         ",server=on,wait=off");
417 
418   qemu_cmd.AddParameter("-mon");
419   qemu_cmd.AddParameter("chardev=charmonitor,id=monitor,mode=control");
420 
421   auto gpu_mode = instance.gpu_mode();
422   if (gpu_mode == kGpuModeDrmVirgl) {
423     qemu_cmd.AddParameter("-display");
424     qemu_cmd.AddParameter("egl-headless");
425 
426     qemu_cmd.AddParameter("-vnc");
427     qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
428   } else if (gpu_mode == kGpuModeGuestSwiftshader ||
429              gpu_mode == kGpuModeGfxstream ||
430              gpu_mode == kGpuModeGfxstreamGuestAngle ||
431              gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
432     qemu_cmd.AddParameter("-vnc");
433     qemu_cmd.AddParameter("127.0.0.1:", instance.qemu_vnc_server_port());
434   } else {
435     qemu_cmd.AddParameter("-display");
436     qemu_cmd.AddParameter("none");
437   }
438 
439   if (instance.hwcomposer() != kHwComposerNone) {
440     auto display_configs = instance.display_configs();
441     CF_EXPECT(display_configs.size() >= 1);
442     auto display_config = display_configs[0];
443 
444     qemu_cmd.AddParameter("-device");
445 
446     std::string gpu_device;
447     if (gpu_mode == kGpuModeGuestSwiftshader || qemu_version.first < 6) {
448       gpu_device = "virtio-gpu-pci";
449     } else if (gpu_mode == kGpuModeDrmVirgl) {
450       gpu_device = "virtio-gpu-gl-pci";
451     } else if (gpu_mode == kGpuModeGfxstream) {
452       gpu_device =
453           "virtio-gpu-rutabaga,x-gfxstream-gles=on,gfxstream-vulkan=on,"
454           "x-gfxstream-composer=on,hostmem=256M";
455     } else if (gpu_mode == kGpuModeGfxstreamGuestAngle ||
456                gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
457       gpu_device =
458           "virtio-gpu-rutabaga,gfxstream-vulkan=on,"
459           "x-gfxstream-composer=on,hostmem=256M";
460 
461       if (gpu_mode == kGpuModeGfxstreamGuestAngleHostSwiftShader) {
462         // See https://github.com/KhronosGroup/Vulkan-Loader.
463         const std::string swiftshader_icd_json =
464             HostUsrSharePath("vulkan/icd.d/vk_swiftshader_icd.json");
465         qemu_cmd.AddEnvironmentVariable("VK_DRIVER_FILES",
466                                         swiftshader_icd_json);
467         qemu_cmd.AddEnvironmentVariable("VK_ICD_FILENAMES",
468                                         swiftshader_icd_json);
469       }
470     }
471 
472     qemu_cmd.AddParameter(
473         gpu_device, ",id=gpu0",
474         fmt::format(",addr={:0>2x}.0", VmManager::kGpuPciSlotNum),
475         ",xres=", display_config.width, ",yres=", display_config.height);
476   }
477 
478   if (!instance.console()) {
479     // In kgdb mode, earlycon is an interactive console, and so early
480     // dmesg will go there instead of the kernel.log. On QEMU, we do this
481     // bit of logic up before the hvc console is set up, so the command line
482     // flags appear in the right order and "append=on" does the right thing
483     if (instance.enable_kernel_log() &&
484         (instance.kgdb() || instance.use_bootloader())) {
485       add_serial_console_ro(instance.kernel_log_pipe_name());
486     }
487   }
488 
489   qemu_cmd.AddParameter("-device");
490   qemu_cmd.AddParameter(
491       "virtio-serial-pci-non-transitional,max_ports=31,id=virtio-serial");
492 
493   // /dev/hvc0 = kernel console
494   // If kernel log is enabled, the virtio-console port will be specified as
495   // a true console for Linux, and kernel messages will be printed there.
496   // Otherwise, the port will still be set up for bootloader and userspace
497   // messages, but the kernel will not print anything here. This keeps our
498   // kernel log event features working. If an alternative "earlycon" boot
499   // console is configured above on a legacy serial port, it will control
500   // the main log until the virtio-console takes over.
501   // (Note that QEMU does not automatically generate console= parameters for
502   //  the bootloader/kernel cmdline, so the control of whether this pipe is
503   //  actually managed by the kernel as a console is handled elsewhere.)
504   add_hvc_ro(instance.kernel_log_pipe_name());
505 
506   // /dev/hvc1 = serial console
507   if (instance.console()) {
508     if (instance.kgdb() || instance.use_bootloader()) {
509       add_serial_console(instance.console_pipe_prefix());
510 
511       // In kgdb mode, we have the interactive console on ttyS0 (both Android's
512       // console and kdb), so we can disable the virtio-console port usually
513       // allocated to Android's serial console, and redirect it to a sink. This
514       // ensures that that the PCI device assignments (and thus sepolicy) don't
515       // have to change
516       add_hvc_sink();
517     } else {
518       add_serial_sink();
519       add_hvc(instance.console_pipe_prefix());
520     }
521   } else {
522     if (instance.kgdb() || instance.use_bootloader()) {
523       // The add_serial_console_ro() call above was applied by the time we reach
524       // this code, so we don't need another add_serial_*() call
525     }
526 
527     // as above, create a fake virtio-console 'sink' port when the serial
528     // console is disabled, so the PCI device ID assignments don't move
529     // around
530     add_hvc_sink();
531   }
532 
533   // /dev/hvc2 = serial logging
534   // Serial port for logcat, redirected to a pipe
535   add_hvc_ro(instance.logcat_pipe_name());
536 
537   // /dev/hvc3 = keymaster (C++ implementation)
538   add_hvc(instance.PerInstanceInternalPath("keymaster_fifo_vm"));
539   // /dev/hvc4 = gatekeeper
540   add_hvc(instance.PerInstanceInternalPath("gatekeeper_fifo_vm"));
541   // /dev/hvc5 = bt
542   if (config.enable_host_bluetooth()) {
543     add_hvc(instance.PerInstanceInternalPath("bt_fifo_vm"));
544   } else {
545     add_hvc_sink();
546   }
547 
548   // /dev/hvc6 = gnss
549   // /dev/hvc7 = location
550   if (instance.enable_gnss_grpc_proxy()) {
551     add_hvc(instance.PerInstanceInternalPath("gnsshvc_fifo_vm"));
552     add_hvc(instance.PerInstanceInternalPath("locationhvc_fifo_vm"));
553   } else {
554     for (auto i = 0; i < 2; i++) {
555       add_hvc_sink();
556     }
557   }
558 
559   /* Added one for confirmation UI.
560    *
561    * b/237452165
562    *
563    * Confirmation UI is not supported with QEMU for now. In order
564    * to not conflict with confirmation UI-related configurations used
565    * w/ Crosvm, we should add one generic avc.
566    *
567    * confui_fifo_vm.{in/out} are created along with the streamer process,
568    * which is not created w/ QEMU.
569    */
570   // /dev/hvc8 = confirmationui
571   add_hvc_sink();
572 
573   // /dev/hvc9 = uwb
574   if (config.enable_host_uwb()) {
575     add_hvc(instance.PerInstanceInternalPath("uwb_fifo_vm"));
576   } else {
577     add_hvc_sink();
578   }
579 
580   // /dev/hvc10 = oemlock
581   add_hvc(instance.PerInstanceInternalPath("oemlock_fifo_vm"));
582 
583   // /dev/hvc11 = keymint (Rust implementation)
584   add_hvc(instance.PerInstanceInternalPath("keymint_fifo_vm"));
585 
586   // /dev/hvc12 = nfc
587   if (config.enable_host_nfc()) {
588     add_hvc(instance.PerInstanceInternalPath("nfc_fifo_vm"));
589   } else {
590     add_hvc_sink();
591   }
592 
593   // sensors_fifo_vm.{in/out} are created along with the streamer process,
594   // which is not created w/ QEMU.
595   // /dev/hvc13 = sensors
596   add_hvc_sink();
597 
598   // /dev/hvc14 = MCU CONTROL
599   if (instance.mcu()["control"]["type"].asString() == "serial") {
600     auto path = instance.PerInstanceInternalPath("mcu");
601     path += "/" + instance.mcu()["control"]["path"].asString();
602     add_hvc_serial(path);
603   } else {
604     add_hvc_sink();
605   }
606 
607   // /dev/hvc15 = MCU UART
608   if (instance.mcu()["uart0"]["type"].asString() == "serial") {
609     auto path = instance.PerInstanceInternalPath("mcu");
610     path += "/" + instance.mcu()["uart0"]["path"].asString();
611     add_hvc_serial(path);
612   } else {
613     add_hvc_sink();
614   }
615 
616   auto disk_num = instance.virtual_disk_paths().size();
617 
618   for (auto i = 0; i < VmManager::kMaxDisks - disk_num; i++) {
619     add_hvc_sink();
620   }
621 
622   CF_EXPECT(
623       hvc_num + disk_num == VmManager::kMaxDisks + VmManager::kDefaultNumHvcs,
624       "HVC count (" << hvc_num << ") + disk count (" << disk_num << ") "
625                     << "is not the expected total of "
626                     << VmManager::kMaxDisks + VmManager::kDefaultNumHvcs
627                     << " devices");
628 
629   CF_EXPECT(VmManager::kMaxDisks >= disk_num,
630             "Provided too many disks (" << disk_num << "), maximum "
631                                         << VmManager::kMaxDisks << "supported");
632   auto readonly = instance.protected_vm() ? ",readonly" : "";
633   size_t i = 0;
634   for (const auto& disk : instance.virtual_disk_paths()) {
635     if (instance.vhost_user_block()) {
636       auto block = CF_EXPECT(VhostUserBlockDevice(config, i, disk));
637       commands.emplace_back(std::move(block.device_cmd));
638       commands.emplace_back(std::move(block.device_logs_cmd));
639       auto socket_path = std::move(block.socket_path);
640       qemu_cmd.AddPrerequisite([socket_path]() -> Result<void> {
641 #ifdef __linux__
642         return WaitForUnixSocketListeningWithoutConnect(socket_path,
643                                                         /*timeoutSec=*/30);
644 #else
645         return CF_ERR("Unhandled check if vhost user block ready.");
646 #endif
647       });
648 
649       qemu_cmd.AddParameter("-chardev");
650       qemu_cmd.AddParameter("socket,id=vhost-user-block-", i,
651                             ",path=", socket_path);
652       qemu_cmd.AddParameter("-device");
653       qemu_cmd.AddParameter(
654           "vhost-user-blk-pci-non-transitional,chardev=vhost-user-block-", i);
655     } else {
656       qemu_cmd.AddParameter("-drive");
657       qemu_cmd.AddParameter("file=", disk, ",if=none,id=drive-virtio-disk", i,
658                             ",aio=threads", readonly);
659       qemu_cmd.AddParameter("-device");
660       qemu_cmd.AddParameter(
661           "virtio-blk-pci-non-transitional,drive=drive-virtio-disk", i,
662           ",id=virtio-disk", i, (i == 0 ? ",bootindex=1" : ""));
663     }
664     ++i;
665   }
666 
667   if (is_x86 && FileExists(instance.pstore_path())) {
668     // QEMU will assign the NVDIMM (ramoops pstore region) 150000000-1501fffff
669     // As we will pass this to ramoops, define this region first so it is always
670     // located at this address. This is currently x86 only.
671     qemu_cmd.AddParameter("-object");
672     qemu_cmd.AddParameter("memory-backend-file,id=objpmem0,share=on,mem-path=",
673                           instance.pstore_path(), ",size=", pstore_size_bytes);
674 
675     qemu_cmd.AddParameter("-device");
676     qemu_cmd.AddParameter("nvdimm,memdev=objpmem0,id=ramoops");
677   }
678 
679   // QEMU does not implement virtio-pmem-pci for ARM64 or RISC-V yet; restore
680   // this when the device has been added
681   if (is_x86) {
682     if (access_kregistry_size_bytes > 0) {
683       qemu_cmd.AddParameter("-object");
684       qemu_cmd.AddParameter(
685           "memory-backend-file,id=objpmem1,share=on,mem-path=",
686           instance.access_kregistry_path(),
687           ",size=", access_kregistry_size_bytes);
688 
689       qemu_cmd.AddParameter("-device");
690       qemu_cmd.AddParameter(
691           "virtio-pmem-pci,disable-legacy=on,memdev=objpmem1,id=pmem0");
692     }
693     if (hwcomposer_pmem_size_bytes > 0) {
694       qemu_cmd.AddParameter("-object");
695       qemu_cmd.AddParameter(
696           "memory-backend-file,id=objpmem2,share=on,mem-path=",
697           instance.hwcomposer_pmem_path(),
698           ",size=", hwcomposer_pmem_size_bytes);
699 
700       qemu_cmd.AddParameter("-device");
701       qemu_cmd.AddParameter(
702           "virtio-pmem-pci,disable-legacy=on,memdev=objpmem2,id=pmem1");
703     }
704   }
705 
706   qemu_cmd.AddParameter("-object");
707   qemu_cmd.AddParameter("rng-random,id=objrng0,filename=/dev/urandom");
708 
709   qemu_cmd.AddParameter("-device");
710   qemu_cmd.AddParameter("virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,",
711                         "max-bytes=1024,period=2000");
712 
713   qemu_cmd.AddParameter("-device");
714   qemu_cmd.AddParameter("virtio-mouse-pci,disable-legacy=on");
715 
716   qemu_cmd.AddParameter("-device");
717   qemu_cmd.AddParameter("virtio-keyboard-pci,disable-legacy=on");
718 
719   // device padding for unsupported "switches" input
720   qemu_cmd.AddParameter("-device");
721   qemu_cmd.AddParameter("virtio-keyboard-pci,disable-legacy=on");
722 
723   auto vhost_net = instance.vhost_net() ? ",vhost=on" : "";
724 
725   qemu_cmd.AddParameter("-device");
726   qemu_cmd.AddParameter("virtio-balloon-pci-non-transitional,id=balloon0");
727 
728   switch (instance.external_network_mode()) {
729     case ExternalNetworkMode::kTap:
730       qemu_cmd.AddParameter("-netdev");
731       qemu_cmd.AddParameter(
732           "tap,id=hostnet0,ifname=", instance.mobile_tap_name(),
733           ",script=no,downscript=no", vhost_net);
734 
735       qemu_cmd.AddParameter("-netdev");
736       qemu_cmd.AddParameter(
737           "tap,id=hostnet1,ifname=", instance.ethernet_tap_name(),
738           ",script=no,downscript=no", vhost_net);
739 
740       if (!config.virtio_mac80211_hwsim()) {
741         qemu_cmd.AddParameter("-netdev");
742         qemu_cmd.AddParameter(
743             "tap,id=hostnet2,ifname=", instance.wifi_tap_name(),
744             ",script=no,downscript=no", vhost_net);
745       }
746       break;
747     case cuttlefish::ExternalNetworkMode::kSlirp: {
748       const std::string net =
749           fmt::format("{}/{}", instance.ril_ipaddr(), instance.ril_prefixlen());
750       const std::string& host = instance.ril_gateway();
751       qemu_cmd.AddParameter("-netdev");
752       // TODO(schuffelen): `dns` needs to match the first `nameserver` in
753       // `/etc/resolv.conf`. Implement something that generalizes beyond gLinux.
754       qemu_cmd.AddParameter("user,id=hostnet0,net=", net, ",host=", host,
755                             ",dns=127.0.0.1");
756 
757       qemu_cmd.AddParameter("-netdev");
758       qemu_cmd.AddParameter("user,id=hostnet1,net=10.0.1.1/24,dns=8.8.4.4");
759 
760       if (!config.virtio_mac80211_hwsim()) {
761         qemu_cmd.AddParameter("-netdev");
762         qemu_cmd.AddParameter("user,id=hostnet2,net=10.0.2.1/24,dns=1.1.1.1");
763       }
764       break;
765     }
766     default:
767       return CF_ERRF("Unexpected net mode {}",
768                      instance.external_network_mode());
769   }
770 
771   // The ordering of virtio-net devices is important. Make sure any change here
772   // is reflected in ethprime u-boot variable
773   qemu_cmd.AddParameter("-device");
774   qemu_cmd.AddParameter(
775       "virtio-net-pci-non-transitional,netdev=hostnet0,id=net0,mac=",
776       instance.mobile_mac());
777   qemu_cmd.AddParameter("-device");
778   qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet1,id=net1,mac=",
779                         instance.ethernet_mac());
780   if (!config.virtio_mac80211_hwsim()) {
781     qemu_cmd.AddParameter("-device");
782     qemu_cmd.AddParameter("virtio-net-pci-non-transitional,netdev=hostnet2,id=net2,mac=",
783                           instance.wifi_mac());
784   }
785 
786   if (is_x86 || is_arm) {
787     qemu_cmd.AddParameter("-cpu");
788     qemu_cmd.AddParameter(IsHostCompatible(arch_) ? "host" : "max");
789   }
790 
791   // Explicitly enable the optional extensions of interest, in case the default
792   // behavior changes upstream.
793   if (is_riscv64) {
794     qemu_cmd.AddParameter("-cpu");
795     qemu_cmd.AddParameter("rv64",
796                           ",v=true,elen=64,vlen=128",
797                           ",zba=true,zbb=true,zbs=true");
798   }
799 
800   qemu_cmd.AddParameter("-msg");
801   qemu_cmd.AddParameter("timestamp=on");
802 
803 #ifdef __linux__
804   qemu_cmd.AddParameter("-device");
805   qemu_cmd.AddParameter("vhost-vsock-pci-non-transitional,guest-cid=",
806                         instance.vsock_guest_cid());
807 #endif
808 
809   qemu_cmd.AddParameter("-device");
810   qemu_cmd.AddParameter("AC97,audiodev=audio_none");
811   qemu_cmd.AddParameter("-audiodev");
812   qemu_cmd.AddParameter("driver=none,id=audio_none");
813 
814   qemu_cmd.AddParameter("-device");
815   qemu_cmd.AddParameter("qemu-xhci,id=xhci");
816 
817   qemu_cmd.AddParameter("-L");
818   qemu_cmd.AddParameter(HostQemuBiosPath());
819 
820   if (is_riscv64) {
821     qemu_cmd.AddParameter("-kernel");
822     qemu_cmd.AddParameter(instance.bootloader());
823   } else if (is_arm) {
824     qemu_cmd.AddParameter("-bios");
825     qemu_cmd.AddParameter(instance.bootloader());
826   } else {
827     qemu_cmd.AddParameter("-drive");
828     qemu_cmd.AddParameter("if=pflash,format=raw,readonly=on,file=",
829                           instance.bootloader());
830     qemu_cmd.AddParameter("-drive");
831     qemu_cmd.AddParameter("if=pflash,format=raw,file=", instance.pflash_path());
832   }
833 
834   if (instance.gdb_port() > 0) {
835     qemu_cmd.AddParameter("-S");
836     qemu_cmd.AddParameter("-gdb");
837     qemu_cmd.AddParameter("tcp::", instance.gdb_port());
838   }
839 
840   commands.emplace_back(std::move(qemu_cmd), true);
841   return commands;
842 }
843 
844 } // namespace vm_manager
845 }  // namespace cuttlefish
846