1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved
4 */
5
6 #include <linux/device.h>
7 #include <linux/module.h>
8 #include <linux/mutex.h>
9 #include <linux/pci.h>
10 #include <linux/pm_runtime.h>
11 #include <linux/types.h>
12 #include <linux/uaccess.h>
13 #include <linux/vfio.h>
14 #include <linux/vfio_pci_core.h>
15 #include <linux/virtio_pci.h>
16 #include <linux/virtio_net.h>
17 #include <linux/virtio_pci_admin.h>
18
19 #include "common.h"
20
21 static int
virtiovf_issue_legacy_rw_cmd(struct virtiovf_pci_core_device * virtvdev,loff_t pos,char __user * buf,size_t count,bool read)22 virtiovf_issue_legacy_rw_cmd(struct virtiovf_pci_core_device *virtvdev,
23 loff_t pos, char __user *buf,
24 size_t count, bool read)
25 {
26 bool msix_enabled =
27 (virtvdev->core_device.irq_type == VFIO_PCI_MSIX_IRQ_INDEX);
28 struct pci_dev *pdev = virtvdev->core_device.pdev;
29 u8 *bar0_buf = virtvdev->bar0_virtual_buf;
30 bool common;
31 u8 offset;
32 int ret;
33
34 common = pos < VIRTIO_PCI_CONFIG_OFF(msix_enabled);
35 /* offset within the relevant configuration area */
36 offset = common ? pos : pos - VIRTIO_PCI_CONFIG_OFF(msix_enabled);
37 mutex_lock(&virtvdev->bar_mutex);
38 if (read) {
39 if (common)
40 ret = virtio_pci_admin_legacy_common_io_read(pdev, offset,
41 count, bar0_buf + pos);
42 else
43 ret = virtio_pci_admin_legacy_device_io_read(pdev, offset,
44 count, bar0_buf + pos);
45 if (ret)
46 goto out;
47 if (copy_to_user(buf, bar0_buf + pos, count))
48 ret = -EFAULT;
49 } else {
50 if (copy_from_user(bar0_buf + pos, buf, count)) {
51 ret = -EFAULT;
52 goto out;
53 }
54
55 if (common)
56 ret = virtio_pci_admin_legacy_common_io_write(pdev, offset,
57 count, bar0_buf + pos);
58 else
59 ret = virtio_pci_admin_legacy_device_io_write(pdev, offset,
60 count, bar0_buf + pos);
61 }
62 out:
63 mutex_unlock(&virtvdev->bar_mutex);
64 return ret;
65 }
66
67 static int
virtiovf_pci_bar0_rw(struct virtiovf_pci_core_device * virtvdev,loff_t pos,char __user * buf,size_t count,bool read)68 virtiovf_pci_bar0_rw(struct virtiovf_pci_core_device *virtvdev,
69 loff_t pos, char __user *buf,
70 size_t count, bool read)
71 {
72 struct vfio_pci_core_device *core_device = &virtvdev->core_device;
73 struct pci_dev *pdev = core_device->pdev;
74 u16 queue_notify;
75 int ret;
76
77 if (!(le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO))
78 return -EIO;
79
80 if (pos + count > virtvdev->bar0_virtual_buf_size)
81 return -EINVAL;
82
83 ret = pm_runtime_resume_and_get(&pdev->dev);
84 if (ret) {
85 pci_info_ratelimited(pdev, "runtime resume failed %d\n", ret);
86 return -EIO;
87 }
88
89 switch (pos) {
90 case VIRTIO_PCI_QUEUE_NOTIFY:
91 if (count != sizeof(queue_notify)) {
92 ret = -EINVAL;
93 goto end;
94 }
95 if (read) {
96 ret = vfio_pci_core_ioread16(core_device, true, &queue_notify,
97 virtvdev->notify_addr);
98 if (ret)
99 goto end;
100 if (copy_to_user(buf, &queue_notify,
101 sizeof(queue_notify))) {
102 ret = -EFAULT;
103 goto end;
104 }
105 } else {
106 if (copy_from_user(&queue_notify, buf, count)) {
107 ret = -EFAULT;
108 goto end;
109 }
110 ret = vfio_pci_core_iowrite16(core_device, true, queue_notify,
111 virtvdev->notify_addr);
112 }
113 break;
114 default:
115 ret = virtiovf_issue_legacy_rw_cmd(virtvdev, pos, buf, count,
116 read);
117 }
118
119 end:
120 pm_runtime_put(&pdev->dev);
121 return ret ? ret : count;
122 }
123
virtiovf_pci_read_config(struct vfio_device * core_vdev,char __user * buf,size_t count,loff_t * ppos)124 static ssize_t virtiovf_pci_read_config(struct vfio_device *core_vdev,
125 char __user *buf, size_t count,
126 loff_t *ppos)
127 {
128 struct virtiovf_pci_core_device *virtvdev = container_of(
129 core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
130 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
131 size_t register_offset;
132 loff_t copy_offset;
133 size_t copy_count;
134 __le32 val32;
135 __le16 val16;
136 u8 val8;
137 int ret;
138
139 ret = vfio_pci_core_read(core_vdev, buf, count, ppos);
140 if (ret < 0)
141 return ret;
142
143 if (vfio_pci_core_range_intersect_range(pos, count, PCI_DEVICE_ID,
144 sizeof(val16), ©_offset,
145 ©_count, ®ister_offset)) {
146 val16 = cpu_to_le16(VIRTIO_TRANS_ID_NET);
147 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset, copy_count))
148 return -EFAULT;
149 }
150
151 if ((le16_to_cpu(virtvdev->pci_cmd) & PCI_COMMAND_IO) &&
152 vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND,
153 sizeof(val16), ©_offset,
154 ©_count, ®ister_offset)) {
155 if (copy_from_user((void *)&val16 + register_offset, buf + copy_offset,
156 copy_count))
157 return -EFAULT;
158 val16 |= cpu_to_le16(PCI_COMMAND_IO);
159 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
160 copy_count))
161 return -EFAULT;
162 }
163
164 if (vfio_pci_core_range_intersect_range(pos, count, PCI_REVISION_ID,
165 sizeof(val8), ©_offset,
166 ©_count, ®ister_offset)) {
167 /* Transional needs to have revision 0 */
168 val8 = 0;
169 if (copy_to_user(buf + copy_offset, &val8, copy_count))
170 return -EFAULT;
171 }
172
173 if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0,
174 sizeof(val32), ©_offset,
175 ©_count, ®ister_offset)) {
176 u32 bar_mask = ~(virtvdev->bar0_virtual_buf_size - 1);
177 u32 pci_base_addr_0 = le32_to_cpu(virtvdev->pci_base_addr_0);
178
179 val32 = cpu_to_le32((pci_base_addr_0 & bar_mask) | PCI_BASE_ADDRESS_SPACE_IO);
180 if (copy_to_user(buf + copy_offset, (void *)&val32 + register_offset, copy_count))
181 return -EFAULT;
182 }
183
184 if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_ID,
185 sizeof(val16), ©_offset,
186 ©_count, ®ister_offset)) {
187 /*
188 * Transitional devices use the PCI subsystem device id as
189 * virtio device id, same as legacy driver always did.
190 */
191 val16 = cpu_to_le16(VIRTIO_ID_NET);
192 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
193 copy_count))
194 return -EFAULT;
195 }
196
197 if (vfio_pci_core_range_intersect_range(pos, count, PCI_SUBSYSTEM_VENDOR_ID,
198 sizeof(val16), ©_offset,
199 ©_count, ®ister_offset)) {
200 val16 = cpu_to_le16(PCI_VENDOR_ID_REDHAT_QUMRANET);
201 if (copy_to_user(buf + copy_offset, (void *)&val16 + register_offset,
202 copy_count))
203 return -EFAULT;
204 }
205
206 return count;
207 }
208
virtiovf_pci_core_read(struct vfio_device * core_vdev,char __user * buf,size_t count,loff_t * ppos)209 ssize_t virtiovf_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
210 size_t count, loff_t *ppos)
211 {
212 struct virtiovf_pci_core_device *virtvdev = container_of(
213 core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
214 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
215 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
216
217 if (!count)
218 return 0;
219
220 if (index == VFIO_PCI_CONFIG_REGION_INDEX)
221 return virtiovf_pci_read_config(core_vdev, buf, count, ppos);
222
223 if (index == VFIO_PCI_BAR0_REGION_INDEX)
224 return virtiovf_pci_bar0_rw(virtvdev, pos, buf, count, true);
225
226 return vfio_pci_core_read(core_vdev, buf, count, ppos);
227 }
228
virtiovf_pci_write_config(struct vfio_device * core_vdev,const char __user * buf,size_t count,loff_t * ppos)229 static ssize_t virtiovf_pci_write_config(struct vfio_device *core_vdev,
230 const char __user *buf, size_t count,
231 loff_t *ppos)
232 {
233 struct virtiovf_pci_core_device *virtvdev = container_of(
234 core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
235 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
236 size_t register_offset;
237 loff_t copy_offset;
238 size_t copy_count;
239
240 if (vfio_pci_core_range_intersect_range(pos, count, PCI_COMMAND,
241 sizeof(virtvdev->pci_cmd),
242 ©_offset, ©_count,
243 ®ister_offset)) {
244 if (copy_from_user((void *)&virtvdev->pci_cmd + register_offset,
245 buf + copy_offset,
246 copy_count))
247 return -EFAULT;
248 }
249
250 if (vfio_pci_core_range_intersect_range(pos, count, PCI_BASE_ADDRESS_0,
251 sizeof(virtvdev->pci_base_addr_0),
252 ©_offset, ©_count,
253 ®ister_offset)) {
254 if (copy_from_user((void *)&virtvdev->pci_base_addr_0 + register_offset,
255 buf + copy_offset,
256 copy_count))
257 return -EFAULT;
258 }
259
260 return vfio_pci_core_write(core_vdev, buf, count, ppos);
261 }
262
virtiovf_pci_core_write(struct vfio_device * core_vdev,const char __user * buf,size_t count,loff_t * ppos)263 ssize_t virtiovf_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
264 size_t count, loff_t *ppos)
265 {
266 struct virtiovf_pci_core_device *virtvdev = container_of(
267 core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
268 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
269 loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
270
271 if (!count)
272 return 0;
273
274 if (index == VFIO_PCI_CONFIG_REGION_INDEX)
275 return virtiovf_pci_write_config(core_vdev, buf, count, ppos);
276
277 if (index == VFIO_PCI_BAR0_REGION_INDEX)
278 return virtiovf_pci_bar0_rw(virtvdev, pos, (char __user *)buf, count, false);
279
280 return vfio_pci_core_write(core_vdev, buf, count, ppos);
281 }
282
virtiovf_pci_ioctl_get_region_info(struct vfio_device * core_vdev,unsigned int cmd,unsigned long arg)283 int virtiovf_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
284 unsigned int cmd, unsigned long arg)
285 {
286 struct virtiovf_pci_core_device *virtvdev = container_of(
287 core_vdev, struct virtiovf_pci_core_device, core_device.vdev);
288 unsigned long minsz = offsetofend(struct vfio_region_info, offset);
289 void __user *uarg = (void __user *)arg;
290 struct vfio_region_info info = {};
291
292 if (copy_from_user(&info, uarg, minsz))
293 return -EFAULT;
294
295 if (info.argsz < minsz)
296 return -EINVAL;
297
298 switch (info.index) {
299 case VFIO_PCI_BAR0_REGION_INDEX:
300 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
301 info.size = virtvdev->bar0_virtual_buf_size;
302 info.flags = VFIO_REGION_INFO_FLAG_READ |
303 VFIO_REGION_INFO_FLAG_WRITE;
304 return copy_to_user(uarg, &info, minsz) ? -EFAULT : 0;
305 default:
306 return vfio_pci_core_ioctl(core_vdev, cmd, arg);
307 }
308 }
309
virtiovf_vfio_pci_core_ioctl(struct vfio_device * core_vdev,unsigned int cmd,unsigned long arg)310 long virtiovf_vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
311 unsigned long arg)
312 {
313 switch (cmd) {
314 case VFIO_DEVICE_GET_REGION_INFO:
315 return virtiovf_pci_ioctl_get_region_info(core_vdev, cmd, arg);
316 default:
317 return vfio_pci_core_ioctl(core_vdev, cmd, arg);
318 }
319 }
320
virtiovf_set_notify_addr(struct virtiovf_pci_core_device * virtvdev)321 static int virtiovf_set_notify_addr(struct virtiovf_pci_core_device *virtvdev)
322 {
323 struct vfio_pci_core_device *core_device = &virtvdev->core_device;
324 int ret;
325
326 /*
327 * Setup the BAR where the 'notify' exists to be used by vfio as well
328 * This will let us mmap it only once and use it when needed.
329 */
330 ret = vfio_pci_core_setup_barmap(core_device,
331 virtvdev->notify_bar);
332 if (ret)
333 return ret;
334
335 virtvdev->notify_addr = core_device->barmap[virtvdev->notify_bar] +
336 virtvdev->notify_offset;
337 return 0;
338 }
339
virtiovf_open_legacy_io(struct virtiovf_pci_core_device * virtvdev)340 int virtiovf_open_legacy_io(struct virtiovf_pci_core_device *virtvdev)
341 {
342 if (!virtvdev->bar0_virtual_buf)
343 return 0;
344
345 /*
346 * Upon close_device() the vfio_pci_core_disable() is called
347 * and will close all the previous mmaps, so it seems that the
348 * valid life cycle for the 'notify' addr is per open/close.
349 */
350 return virtiovf_set_notify_addr(virtvdev);
351 }
352
virtiovf_get_device_config_size(unsigned short device)353 static int virtiovf_get_device_config_size(unsigned short device)
354 {
355 /* Network card */
356 return offsetofend(struct virtio_net_config, status);
357 }
358
virtiovf_read_notify_info(struct virtiovf_pci_core_device * virtvdev)359 static int virtiovf_read_notify_info(struct virtiovf_pci_core_device *virtvdev)
360 {
361 u64 offset;
362 int ret;
363 u8 bar;
364
365 ret = virtio_pci_admin_legacy_io_notify_info(virtvdev->core_device.pdev,
366 VIRTIO_ADMIN_CMD_NOTIFY_INFO_FLAGS_OWNER_MEM,
367 &bar, &offset);
368 if (ret)
369 return ret;
370
371 virtvdev->notify_bar = bar;
372 virtvdev->notify_offset = offset;
373 return 0;
374 }
375
virtiovf_bar0_exists(struct pci_dev * pdev)376 static bool virtiovf_bar0_exists(struct pci_dev *pdev)
377 {
378 struct resource *res = pdev->resource;
379
380 return res->flags;
381 }
382
virtiovf_support_legacy_io(struct pci_dev * pdev)383 bool virtiovf_support_legacy_io(struct pci_dev *pdev)
384 {
385 return virtio_pci_admin_has_legacy_io(pdev) && !virtiovf_bar0_exists(pdev);
386 }
387
virtiovf_init_legacy_io(struct virtiovf_pci_core_device * virtvdev)388 int virtiovf_init_legacy_io(struct virtiovf_pci_core_device *virtvdev)
389 {
390 struct pci_dev *pdev = virtvdev->core_device.pdev;
391 int ret;
392
393 ret = virtiovf_read_notify_info(virtvdev);
394 if (ret)
395 return ret;
396
397 virtvdev->bar0_virtual_buf_size = VIRTIO_PCI_CONFIG_OFF(true) +
398 virtiovf_get_device_config_size(pdev->device);
399 BUILD_BUG_ON(!is_power_of_2(virtvdev->bar0_virtual_buf_size));
400 virtvdev->bar0_virtual_buf = kzalloc(virtvdev->bar0_virtual_buf_size,
401 GFP_KERNEL);
402 if (!virtvdev->bar0_virtual_buf)
403 return -ENOMEM;
404 mutex_init(&virtvdev->bar_mutex);
405 return 0;
406 }
407
virtiovf_release_legacy_io(struct virtiovf_pci_core_device * virtvdev)408 void virtiovf_release_legacy_io(struct virtiovf_pci_core_device *virtvdev)
409 {
410 kfree(virtvdev->bar0_virtual_buf);
411 }
412
virtiovf_legacy_io_reset_done(struct pci_dev * pdev)413 void virtiovf_legacy_io_reset_done(struct pci_dev *pdev)
414 {
415 struct virtiovf_pci_core_device *virtvdev = dev_get_drvdata(&pdev->dev);
416
417 virtvdev->pci_cmd = 0;
418 }
419