1 // Copyright 2019 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 use std::cmp::min;
6 use std::fmt;
7 use std::fmt::Display;
8 use std::mem;
9 use std::sync::Arc;
10 use std::sync::Weak;
11
12 use base::debug;
13 use base::error;
14 use base::info;
15 use base::warn;
16 use base::Error as SysError;
17 use base::Event;
18 use bit_field::Error as BitFieldError;
19 use remain::sorted;
20 use sync::Mutex;
21 use thiserror::Error;
22 use usb_util::TransferStatus;
23 use usb_util::UsbRequestSetup;
24 use vm_memory::GuestMemory;
25 use vm_memory::GuestMemoryError;
26
27 use super::device_slot::DeviceSlot;
28 use super::interrupter::Error as InterrupterError;
29 use super::interrupter::Interrupter;
30 use super::scatter_gather_buffer::Error as BufferError;
31 use super::scatter_gather_buffer::ScatterGatherBuffer;
32 use super::usb_hub::Error as HubError;
33 use super::usb_hub::UsbPort;
34 use super::xhci_abi::AddressedTrb;
35 use super::xhci_abi::Error as TrbError;
36 use super::xhci_abi::EventDataTrb;
37 use super::xhci_abi::SetupStageTrb;
38 use super::xhci_abi::TransferDescriptor;
39 use super::xhci_abi::TrbCast;
40 use super::xhci_abi::TrbCompletionCode;
41 use super::xhci_abi::TrbType;
42 use super::xhci_regs::MAX_INTERRUPTER;
43
44 #[sorted]
45 #[derive(Error, Debug)]
46 pub enum Error {
47 #[error("unexpected trb type: {0:?}")]
48 BadTrbType(TrbType),
49 #[error("cannot cast trb: {0}")]
50 CastTrb(TrbError),
51 #[error("cannot create transfer buffer: {0}")]
52 CreateBuffer(BufferError),
53 #[error("cannot detach from port: {0}")]
54 DetachPort(HubError),
55 #[error("failed to halt the endpoint: {0}")]
56 HaltEndpoint(u8),
57 #[error("failed to read guest memory: {0}")]
58 ReadGuestMemory(GuestMemoryError),
59 #[error("cannot send interrupt: {0}")]
60 SendInterrupt(InterrupterError),
61 #[error("failed to submit transfer to backend")]
62 SubmitTransfer,
63 #[error("cannot get transfer length: {0}")]
64 TransferLength(TrbError),
65 #[error("cannot get trb type: {0}")]
66 TrbType(BitFieldError),
67 #[error("cannot write completion event: {0}")]
68 WriteCompletionEvent(SysError),
69 #[error("failed to write guest memory: {0}")]
70 WriteGuestMemory(GuestMemoryError),
71 }
72
73 type Result<T> = std::result::Result<T, Error>;
74
75 /// Type of usb endpoints.
76 #[derive(PartialEq, Eq, Clone, Copy, Debug)]
77 pub enum TransferDirection {
78 In,
79 Out,
80 Control,
81 }
82
83 /// Current state of xhci transfer.
84 pub enum XhciTransferState {
85 Created,
86 /// When transfer is submitted, it will contain a transfer callback, which should be invoked
87 /// when the transfer is cancelled.
88 Submitted {
89 cancel_callback: Box<dyn FnOnce() + Send>,
90 },
91 Cancelling,
92 Cancelled,
93 Completed,
94 }
95
96 impl XhciTransferState {
97 /// Try to cancel this transfer, if it's possible.
try_cancel(&mut self)98 pub fn try_cancel(&mut self) {
99 match mem::replace(self, XhciTransferState::Created) {
100 XhciTransferState::Submitted { cancel_callback } => {
101 *self = XhciTransferState::Cancelling;
102 cancel_callback();
103 }
104 XhciTransferState::Cancelling => {
105 error!("Another cancellation is already issued.");
106 }
107 _ => {
108 *self = XhciTransferState::Cancelled;
109 }
110 }
111 }
112 }
113
114 /// Type of a transfer received handled by transfer ring.
115 pub enum XhciTransferType {
116 // Normal means bulk transfer or interrupt transfer, depending on endpoint type.
117 // See spec 4.11.2.1.
118 Normal,
119 // See usb spec for setup stage, data stage and status stage,
120 // see xHCI spec 4.11.2.2 for corresponding trbs.
121 SetupStage,
122 DataStage,
123 StatusStage,
124 // See xHCI spec 4.11.2.3.
125 Isochronous,
126 // See xHCI spec 6.4.1.4.
127 Noop,
128 }
129
130 impl Display for XhciTransferType {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result131 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
132 use self::XhciTransferType::*;
133
134 match self {
135 Normal => write!(f, "Normal"),
136 SetupStage => write!(f, "SetupStage"),
137 DataStage => write!(f, "DataStage"),
138 StatusStage => write!(f, "StatusStage"),
139 Isochronous => write!(f, "Isochronous"),
140 Noop => write!(f, "Noop"),
141 }
142 }
143 }
144
145 /// Xhci Transfer manager holds reference to all ongoing transfers. Can cancel them all if
146 /// needed.
147 #[derive(Clone)]
148 pub struct XhciTransferManager {
149 transfers: Arc<Mutex<Vec<Weak<Mutex<XhciTransferState>>>>>,
150 device_slot: Weak<DeviceSlot>,
151 }
152
153 impl XhciTransferManager {
154 /// Create a new manager.
new(device_slot: Weak<DeviceSlot>) -> XhciTransferManager155 pub fn new(device_slot: Weak<DeviceSlot>) -> XhciTransferManager {
156 XhciTransferManager {
157 transfers: Arc::new(Mutex::new(Vec::new())),
158 device_slot,
159 }
160 }
161
162 /// Build a new XhciTransfer. Endpoint id is the id in xHCI device slot.
create_transfer( &self, mem: GuestMemory, port: Arc<UsbPort>, interrupter: Arc<Mutex<Interrupter>>, slot_id: u8, endpoint_id: u8, transfer_trbs: TransferDescriptor, completion_event: Event, stream_id: Option<u16>, ) -> XhciTransfer163 pub fn create_transfer(
164 &self,
165 mem: GuestMemory,
166 port: Arc<UsbPort>,
167 interrupter: Arc<Mutex<Interrupter>>,
168 slot_id: u8,
169 endpoint_id: u8,
170 transfer_trbs: TransferDescriptor,
171 completion_event: Event,
172 stream_id: Option<u16>,
173 ) -> XhciTransfer {
174 assert!(!transfer_trbs.is_empty());
175 let transfer_dir = {
176 if endpoint_id == 0 {
177 TransferDirection::Control
178 } else if (endpoint_id % 2) == 0 {
179 TransferDirection::Out
180 } else {
181 TransferDirection::In
182 }
183 };
184 let t = XhciTransfer {
185 manager: self.clone(),
186 state: Arc::new(Mutex::new(XhciTransferState::Created)),
187 mem,
188 port,
189 interrupter,
190 transfer_completion_event: completion_event,
191 slot_id,
192 endpoint_id,
193 transfer_dir,
194 transfer_trbs,
195 device_slot: self.device_slot.clone(),
196 stream_id,
197 };
198 self.transfers.lock().push(Arc::downgrade(&t.state));
199 t
200 }
201
202 /// Cancel all current transfers.
cancel_all(&self)203 pub fn cancel_all(&self) {
204 self.transfers.lock().iter().for_each(|t| {
205 let state = match t.upgrade() {
206 Some(state) => state,
207 None => {
208 error!("transfer is already cancelled or finished");
209 return;
210 }
211 };
212 state.lock().try_cancel();
213 });
214 }
215
remove_transfer(&self, t: &Arc<Mutex<XhciTransferState>>)216 fn remove_transfer(&self, t: &Arc<Mutex<XhciTransferState>>) {
217 let mut transfers = self.transfers.lock();
218 match transfers.iter().position(|wt| match wt.upgrade() {
219 Some(wt) => Arc::ptr_eq(&wt, t),
220 None => false,
221 }) {
222 None => error!("attempted to remove unknow transfer"),
223 Some(i) => {
224 transfers.swap_remove(i);
225 }
226 }
227 }
228 }
229
230 impl Default for XhciTransferManager {
default() -> Self231 fn default() -> Self {
232 Self::new(Weak::new())
233 }
234 }
235
236 /// Xhci transfer denotes a transfer initiated by guest os driver. It will be submitted to a
237 /// XhciBackendDevice.
238 pub struct XhciTransfer {
239 manager: XhciTransferManager,
240 state: Arc<Mutex<XhciTransferState>>,
241 mem: GuestMemory,
242 port: Arc<UsbPort>,
243 interrupter: Arc<Mutex<Interrupter>>,
244 slot_id: u8,
245 // id of endpoint in device slot.
246 endpoint_id: u8,
247 transfer_dir: TransferDirection,
248 transfer_trbs: TransferDescriptor,
249 transfer_completion_event: Event,
250 device_slot: Weak<DeviceSlot>,
251 stream_id: Option<u16>,
252 }
253
254 impl Drop for XhciTransfer {
drop(&mut self)255 fn drop(&mut self) {
256 self.manager.remove_transfer(&self.state);
257 }
258 }
259
260 impl fmt::Debug for XhciTransfer {
fmt(&self, f: &mut fmt::Formatter) -> fmt::Result261 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
262 write!(
263 f,
264 "xhci_transfer slot id: {}, endpoint id {}, transfer_dir {:?}, transfer_trbs {:?}",
265 self.slot_id, self.endpoint_id, self.transfer_dir, self.transfer_trbs
266 )
267 }
268 }
269
270 impl XhciTransfer {
271 /// Get state of this transfer.
state(&self) -> &Arc<Mutex<XhciTransferState>>272 pub fn state(&self) -> &Arc<Mutex<XhciTransferState>> {
273 &self.state
274 }
275
276 /// Get transfer type.
get_transfer_type(&self) -> Result<XhciTransferType>277 pub fn get_transfer_type(&self) -> Result<XhciTransferType> {
278 // We can figure out transfer type from the first trb.
279 // See transfer descriptor description in xhci spec for more details.
280 match self.transfer_trbs[0]
281 .trb
282 .get_trb_type()
283 .map_err(Error::TrbType)?
284 {
285 TrbType::Normal => Ok(XhciTransferType::Normal),
286 TrbType::SetupStage => Ok(XhciTransferType::SetupStage),
287 TrbType::DataStage => Ok(XhciTransferType::DataStage),
288 TrbType::StatusStage => Ok(XhciTransferType::StatusStage),
289 TrbType::Isoch => Ok(XhciTransferType::Isochronous),
290 TrbType::Noop => Ok(XhciTransferType::Noop),
291 t => Err(Error::BadTrbType(t)),
292 }
293 }
294
295 /// Create a scatter gather buffer for the given xhci transfer
create_buffer(&self) -> Result<ScatterGatherBuffer>296 pub fn create_buffer(&self) -> Result<ScatterGatherBuffer> {
297 ScatterGatherBuffer::new(self.mem.clone(), self.transfer_trbs.clone())
298 .map_err(Error::CreateBuffer)
299 }
300
301 /// Create a usb request setup for the control transfer buffer
create_usb_request_setup(&self) -> Result<UsbRequestSetup>302 pub fn create_usb_request_setup(&self) -> Result<UsbRequestSetup> {
303 let trb = self.transfer_trbs[0]
304 .trb
305 .checked_cast::<SetupStageTrb>()
306 .map_err(Error::CastTrb)?;
307 Ok(UsbRequestSetup::new(
308 trb.get_request_type(),
309 trb.get_request(),
310 trb.get_value(),
311 trb.get_index(),
312 trb.get_length(),
313 ))
314 }
315
316 /// Get endpoint number.
get_endpoint_number(&self) -> u8317 pub fn get_endpoint_number(&self) -> u8 {
318 // See spec 4.5.1 for dci.
319 self.endpoint_id / 2
320 }
321
322 /// get transfer direction.
get_transfer_dir(&self) -> TransferDirection323 pub fn get_transfer_dir(&self) -> TransferDirection {
324 self.transfer_dir
325 }
326
327 /// get stream id.
get_stream_id(&self) -> Option<u16>328 pub fn get_stream_id(&self) -> Option<u16> {
329 self.stream_id
330 }
331
332 /// This functions should be invoked when transfer is completed (or failed).
on_transfer_complete( &self, status: &TransferStatus, bytes_transferred: u32, ) -> Result<()>333 pub fn on_transfer_complete(
334 &self,
335 status: &TransferStatus,
336 bytes_transferred: u32,
337 ) -> Result<()> {
338 match status {
339 TransferStatus::NoDevice => {
340 info!("xhci: device disconnected, detaching from port");
341 // If the device is gone, we don't need to send transfer completion event, cause we
342 // are going to destroy everything related to this device anyway.
343 return match self.port.detach() {
344 Ok(()) => Ok(()),
345 // It's acceptable for the port to be already disconnected
346 // as asynchronous transfer completions are processed.
347 Err(HubError::AlreadyDetached(_e)) => Ok(()),
348 Err(e) => Err(Error::DetachPort(e)),
349 };
350 }
351 TransferStatus::Cancelled => {
352 // TODO(jkwang) According to the spec, we should send a stopped event here. But
353 // kernel driver does not do anything meaningful when it sees a stopped event.
354 return self
355 .transfer_completion_event
356 .signal()
357 .map_err(Error::WriteCompletionEvent);
358 }
359 TransferStatus::Completed => {
360 self.transfer_completion_event
361 .signal()
362 .map_err(Error::WriteCompletionEvent)?;
363 }
364 TransferStatus::Stalled => {
365 warn!("xhci: endpoint is stalled. set state to Halted");
366 if let Some(device_slot) = self.device_slot.upgrade() {
367 device_slot
368 .halt_endpoint(self.endpoint_id)
369 .map_err(|_| Error::HaltEndpoint(self.endpoint_id))?;
370 }
371 self.transfer_completion_event
372 .signal()
373 .map_err(Error::WriteCompletionEvent)?;
374 }
375 _ => {
376 // Transfer failed, we are not handling this correctly yet. Guest kernel might see
377 // short packets for in transfer and might think control transfer is successful. It
378 // will eventually find out device is in a wrong state.
379 self.transfer_completion_event
380 .signal()
381 .map_err(Error::WriteCompletionEvent)?;
382 }
383 }
384
385 let mut edtla: u32 = 0;
386 // As noted in xHCI spec 4.11.3.1
387 // Transfer Event TRB only occurs under the following conditions:
388 // 1. If the Interrupt On Completion flag is set.
389 // 2. When a short transfer occurs during the execution of a Transfer TRB and the
390 // Interrupt-on-Short Packet flag is set.
391 // 3. If an error occurs during the execution of a Transfer TRB.
392 for atrb in &self.transfer_trbs {
393 edtla += atrb.trb.transfer_length().map_err(Error::TransferLength)?;
394 if atrb.trb.interrupt_on_completion()
395 || (atrb.trb.interrupt_on_short_packet() && edtla > bytes_transferred)
396 {
397 // For details about event data trb and EDTLA, see spec 4.11.5.2.
398 if atrb.trb.get_trb_type().map_err(Error::TrbType)? == TrbType::EventData {
399 let tlength = min(edtla, bytes_transferred);
400 self.interrupter
401 .lock()
402 .send_transfer_event_trb(
403 TrbCompletionCode::Success,
404 atrb.trb
405 .cast::<EventDataTrb>()
406 .map_err(Error::CastTrb)?
407 .get_event_data(),
408 tlength,
409 true,
410 self.slot_id,
411 self.endpoint_id,
412 )
413 .map_err(Error::SendInterrupt)?;
414 } else if *status == TransferStatus::Stalled {
415 debug!("xhci: on transfer complete stalled");
416 let residual_transfer_length = edtla - bytes_transferred;
417 self.interrupter
418 .lock()
419 .send_transfer_event_trb(
420 TrbCompletionCode::StallError,
421 atrb.gpa,
422 residual_transfer_length,
423 true,
424 self.slot_id,
425 self.endpoint_id,
426 )
427 .map_err(Error::SendInterrupt)?;
428 } else {
429 // For Short Transfer details, see xHCI spec 4.10.1.1.
430 if edtla > bytes_transferred {
431 debug!("xhci: on transfer complete short packet");
432 let residual_transfer_length = edtla - bytes_transferred;
433 self.interrupter
434 .lock()
435 .send_transfer_event_trb(
436 TrbCompletionCode::ShortPacket,
437 atrb.gpa,
438 residual_transfer_length,
439 true,
440 self.slot_id,
441 self.endpoint_id,
442 )
443 .map_err(Error::SendInterrupt)?;
444 } else {
445 debug!("xhci: on transfer complete success");
446 self.interrupter
447 .lock()
448 .send_transfer_event_trb(
449 TrbCompletionCode::Success,
450 atrb.gpa,
451 0, // transfer length
452 true,
453 self.slot_id,
454 self.endpoint_id,
455 )
456 .map_err(Error::SendInterrupt)?;
457 }
458 }
459 }
460 }
461 Ok(())
462 }
463
464 /// Send this transfer to backend if it's a valid transfer.
send_to_backend_if_valid(self) -> Result<()>465 pub fn send_to_backend_if_valid(self) -> Result<()> {
466 if self.validate_transfer()? {
467 // Backend should invoke on transfer complete when transfer is completed.
468 let port = self.port.clone();
469 let mut backend = port.backend_device();
470 match &mut *backend {
471 Some(backend) => backend
472 .lock()
473 .submit_xhci_transfer(self)
474 .map_err(|_| Error::SubmitTransfer)?,
475 None => {
476 error!("backend is already disconnected");
477 self.transfer_completion_event
478 .signal()
479 .map_err(Error::WriteCompletionEvent)?;
480 }
481 }
482 } else {
483 error!("invalid td on transfer ring");
484 self.transfer_completion_event
485 .signal()
486 .map_err(Error::WriteCompletionEvent)?;
487 }
488 Ok(())
489 }
490
491 // Check each trb in the transfer descriptor for invalid or out of bounds
492 // parameters. Returns true iff the transfer descriptor is valid.
validate_transfer(&self) -> Result<bool>493 fn validate_transfer(&self) -> Result<bool> {
494 let mut valid = true;
495 for atrb in &self.transfer_trbs {
496 if !trb_is_valid(atrb) {
497 self.interrupter
498 .lock()
499 .send_transfer_event_trb(
500 TrbCompletionCode::TrbError,
501 atrb.gpa,
502 0,
503 false,
504 self.slot_id,
505 self.endpoint_id,
506 )
507 .map_err(Error::SendInterrupt)?;
508 valid = false;
509 }
510 }
511 Ok(valid)
512 }
513 }
514
trb_is_valid(atrb: &AddressedTrb) -> bool515 fn trb_is_valid(atrb: &AddressedTrb) -> bool {
516 let can_be_in_transfer_ring = match atrb.trb.can_be_in_transfer_ring() {
517 Ok(v) => v,
518 Err(e) => {
519 error!("unknown error {:?}", e);
520 return false;
521 }
522 };
523 can_be_in_transfer_ring && (atrb.trb.interrupter_target() < MAX_INTERRUPTER)
524 }
525