1 /*
2 * The PCI Utilities -- Obtain the margin information of the Link
3 *
4 * Copyright (c) 2023-2024 KNS Group LLC (YADRO)
5 *
6 * Can be freely distributed and used under the terms of the GNU GPL v2+.
7 *
8 * SPDX-License-Identifier: GPL-2.0-or-later
9 */
10
11 #include <errno.h>
12 #include <stdlib.h>
13 #include <time.h>
14
15 #include "lmr.h"
16
17 #ifdef PCI_OS_DJGPP
18 #include <unistd.h>
19 #endif
20
21 /* Macro helpers for Margining command parsing */
22
23 typedef u16 margin_cmd;
24
25 /* Margining command parsing */
26
27 #define LMR_CMD_RECVN MASK(2, 0)
28 #define LMR_CMD_TYPE MASK(5, 3)
29 #define LMR_CMD_PAYLOAD MASK(15, 8)
30
31 // Payload parsing
32
33 // Report Capabilities
34 #define LMR_PLD_VOLT_SUPPORT BIT(8)
35 #define LMR_PLD_IND_U_D_VOLT BIT(9)
36 #define LMR_PLD_IND_L_R_TIM BIT(10)
37 #define LMR_PLD_SAMPLE_REPORT_METHOD BIT(11)
38 #define LMR_PLD_IND_ERR_SAMPLER BIT(12)
39
40 #define LMR_PLD_MAX_T_STEPS MASK(13, 8)
41 #define LMR_PLD_MAX_V_STEPS MASK(14, 8)
42 #define LMR_PLD_MAX_OFFSET MASK(14, 8)
43 #define LMR_PLD_MAX_LANES MASK(12, 8)
44 #define LMR_PLD_SAMPLE_RATE MASK(13, 8)
45
46 // Timing Step
47 #define LMR_PLD_MARGIN_T_STEPS MASK(13, 8)
48 #define LMR_PLD_T_GO_LEFT BIT(14)
49
50 // Voltage Timing
51 #define LMR_PLD_MARGIN_V_STEPS MASK(14, 8)
52 #define LMR_PLD_V_GO_DOWN BIT(15)
53
54 // Step Response
55 #define LMR_PLD_ERR_CNT MASK(13, 8)
56 #define LMR_PLD_MARGIN_STS MASK(15, 14)
57
58 /* Address calc macro for Lanes Margining registers */
59
60 #define LMR_LANE_CTRL(lmr_cap_addr, lane) ((lmr_cap_addr) + 8 + 4 * (lane))
61 #define LMR_LANE_STATUS(lmr_cap_addr, lane) ((lmr_cap_addr) + 10 + 4 * (lane))
62
63 /* Margining Commands */
64
65 #define MARG_TIM(go_left, step, recvn) margin_make_cmd(((go_left) << 6) | (step), 3, recvn)
66 #define MARG_VOLT(go_down, step, recvn) margin_make_cmd(((go_down) << 7) | (step), 4, recvn)
67
68 // Report commands
69 #define REPORT_CAPS(recvn) margin_make_cmd(0x88, 1, recvn)
70 #define REPORT_VOL_STEPS(recvn) margin_make_cmd(0x89, 1, recvn)
71 #define REPORT_TIM_STEPS(recvn) margin_make_cmd(0x8A, 1, recvn)
72 #define REPORT_TIM_OFFSET(recvn) margin_make_cmd(0x8B, 1, recvn)
73 #define REPORT_VOL_OFFSET(recvn) margin_make_cmd(0x8C, 1, recvn)
74 #define REPORT_SAMPL_RATE_V(recvn) margin_make_cmd(0x8D, 1, recvn)
75 #define REPORT_SAMPL_RATE_T(recvn) margin_make_cmd(0x8E, 1, recvn)
76 #define REPORT_SAMPLE_CNT(recvn) margin_make_cmd(0x8F, 1, recvn)
77 #define REPORT_MAX_LANES(recvn) margin_make_cmd(0x90, 1, recvn)
78
79 // Set commands
80 #define NO_COMMAND margin_make_cmd(0x9C, 7, 0)
81 #define CLEAR_ERROR_LOG(recvn) margin_make_cmd(0x55, 2, recvn)
82 #define GO_TO_NORMAL_SETTINGS(recvn) margin_make_cmd(0xF, 2, recvn)
83 #define SET_ERROR_LIMIT(error_limit, recvn) margin_make_cmd(0xC0 | (error_limit), 2, recvn)
84
85 static int
msleep(long msec)86 msleep(long msec)
87 {
88 #if defined(PCI_OS_WINDOWS)
89 Sleep(msec);
90 return 0;
91 #elif defined(PCI_OS_DJGPP)
92 if (msec * 1000 < 11264)
93 usleep(11264);
94 else
95 usleep(msec * 1000);
96 return 0;
97 #else
98 struct timespec ts;
99 int res;
100
101 if (msec < 0)
102 {
103 errno = EINVAL;
104 return -1;
105 }
106
107 ts.tv_sec = msec / 1000;
108 ts.tv_nsec = (msec % 1000) * 1000000;
109
110 do
111 {
112 res = nanosleep(&ts, &ts);
113 } while (res && errno == EINTR);
114
115 return res;
116 #endif
117 }
118
119 static margin_cmd
margin_make_cmd(u8 payload,u8 type,u8 recvn)120 margin_make_cmd(u8 payload, u8 type, u8 recvn)
121 {
122 return SET_REG_MASK(0, LMR_CMD_PAYLOAD, payload) | SET_REG_MASK(0, LMR_CMD_TYPE, type)
123 | SET_REG_MASK(0, LMR_CMD_RECVN, recvn);
124 }
125
126 static bool
margin_set_cmd(struct margin_dev * dev,u8 lane,margin_cmd cmd)127 margin_set_cmd(struct margin_dev *dev, u8 lane, margin_cmd cmd)
128 {
129 pci_write_word(dev->dev, LMR_LANE_CTRL(dev->lmr_cap_addr, lane), cmd);
130 msleep(10);
131 return pci_read_word(dev->dev, LMR_LANE_STATUS(dev->lmr_cap_addr, lane)) == cmd;
132 }
133
134 static bool
margin_report_cmd(struct margin_dev * dev,u8 lane,margin_cmd cmd,margin_cmd * result)135 margin_report_cmd(struct margin_dev *dev, u8 lane, margin_cmd cmd, margin_cmd *result)
136 {
137 pci_write_word(dev->dev, LMR_LANE_CTRL(dev->lmr_cap_addr, lane), cmd);
138 msleep(10);
139 *result = pci_read_word(dev->dev, LMR_LANE_STATUS(dev->lmr_cap_addr, lane));
140 return GET_REG_MASK(*result, LMR_CMD_TYPE) == GET_REG_MASK(cmd, LMR_CMD_TYPE)
141 && GET_REG_MASK(*result, LMR_CMD_RECVN) == GET_REG_MASK(cmd, LMR_CMD_RECVN)
142 && margin_set_cmd(dev, lane, NO_COMMAND);
143 }
144
145 static void
margin_apply_hw_quirks(struct margin_recv * recv,struct margin_link_args * args)146 margin_apply_hw_quirks(struct margin_recv *recv, struct margin_link_args *args)
147 {
148 switch (recv->dev->hw)
149 {
150 case MARGIN_ICE_LAKE_RC:
151 if (recv->recvn == 1)
152 {
153 recv->params->volt_offset = 12;
154 args->recv_args[recv->recvn - 1].t.one_side_is_whole = true;
155 args->recv_args[recv->recvn - 1].t.valid = true;
156 }
157 break;
158 default:
159 break;
160 }
161 }
162
163 static bool
read_params_internal(struct margin_dev * dev,u8 recvn,bool lane_reversal,struct margin_params * params)164 read_params_internal(struct margin_dev *dev, u8 recvn, bool lane_reversal,
165 struct margin_params *params)
166 {
167 margin_cmd resp;
168 u8 lane = lane_reversal ? dev->max_width - 1 : 0;
169 margin_set_cmd(dev, lane, NO_COMMAND);
170 bool status = margin_report_cmd(dev, lane, REPORT_CAPS(recvn), &resp);
171 if (status)
172 {
173 params->volt_support = GET_REG_MASK(resp, LMR_PLD_VOLT_SUPPORT);
174 params->ind_up_down_volt = GET_REG_MASK(resp, LMR_PLD_IND_U_D_VOLT);
175 params->ind_left_right_tim = GET_REG_MASK(resp, LMR_PLD_IND_L_R_TIM);
176 params->sample_report_method = GET_REG_MASK(resp, LMR_PLD_SAMPLE_REPORT_METHOD);
177 params->ind_error_sampler = GET_REG_MASK(resp, LMR_PLD_IND_ERR_SAMPLER);
178 status = margin_report_cmd(dev, lane, REPORT_VOL_STEPS(recvn), &resp);
179 }
180 if (status)
181 {
182 params->volt_steps = GET_REG_MASK(resp, LMR_PLD_MAX_V_STEPS);
183 status = margin_report_cmd(dev, lane, REPORT_TIM_STEPS(recvn), &resp);
184 }
185 if (status)
186 {
187 params->timing_steps = GET_REG_MASK(resp, LMR_PLD_MAX_T_STEPS);
188 status = margin_report_cmd(dev, lane, REPORT_TIM_OFFSET(recvn), &resp);
189 }
190 if (status)
191 {
192 params->timing_offset = GET_REG_MASK(resp, LMR_PLD_MAX_OFFSET);
193 status = margin_report_cmd(dev, lane, REPORT_VOL_OFFSET(recvn), &resp);
194 }
195 if (status)
196 {
197 params->volt_offset = GET_REG_MASK(resp, LMR_PLD_MAX_OFFSET);
198 status = margin_report_cmd(dev, lane, REPORT_SAMPL_RATE_V(recvn), &resp);
199 }
200 if (status)
201 {
202 params->sample_rate_v = GET_REG_MASK(resp, LMR_PLD_SAMPLE_RATE);
203 status = margin_report_cmd(dev, lane, REPORT_SAMPL_RATE_T(recvn), &resp);
204 }
205 if (status)
206 {
207 params->sample_rate_t = GET_REG_MASK(resp, LMR_PLD_SAMPLE_RATE);
208 status = margin_report_cmd(dev, lane, REPORT_MAX_LANES(recvn), &resp);
209 }
210 if (status)
211 params->max_lanes = GET_REG_MASK(resp, LMR_PLD_MAX_LANES);
212 return status;
213 }
214
215 /* Margin all lanes_n lanes simultaneously */
216 static void
margin_test_lanes(struct margin_lanes_data arg)217 margin_test_lanes(struct margin_lanes_data arg)
218 {
219 u8 steps_done = 0;
220 margin_cmd lane_status;
221 u8 marg_type;
222 margin_cmd step_cmd;
223 bool timing = (arg.dir == TIM_LEFT || arg.dir == TIM_RIGHT);
224
225 if (timing)
226 {
227 marg_type = 3;
228 step_cmd = MARG_TIM(arg.dir == TIM_LEFT, steps_done, arg.recv->recvn);
229 }
230 else
231 {
232 marg_type = 4;
233 step_cmd = MARG_VOLT(arg.dir == VOLT_DOWN, steps_done, arg.recv->recvn);
234 }
235
236 bool failed_lanes[32] = { 0 };
237 u8 alive_lanes = arg.lanes_n;
238
239 for (int i = 0; i < arg.lanes_n; i++)
240 {
241 margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
242 margin_set_cmd(arg.recv->dev, arg.results[i].lane,
243 SET_ERROR_LIMIT(arg.recv->error_limit, arg.recv->recvn));
244 margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
245 arg.results[i].steps[arg.dir] = arg.steps_lane_total;
246 arg.results[i].statuses[arg.dir] = MARGIN_THR;
247 }
248
249 while (alive_lanes > 0 && steps_done < arg.steps_lane_total)
250 {
251 alive_lanes = 0;
252 steps_done++;
253 if (timing)
254 step_cmd = SET_REG_MASK(step_cmd, LMR_PLD_MARGIN_T_STEPS, steps_done);
255 else
256 step_cmd = SET_REG_MASK(step_cmd, LMR_PLD_MARGIN_V_STEPS, steps_done);
257
258 for (int i = 0; i < arg.lanes_n; i++)
259 {
260 if (!failed_lanes[i])
261 {
262 alive_lanes++;
263 int ctrl_addr = LMR_LANE_CTRL(arg.recv->dev->lmr_cap_addr, arg.results[i].lane);
264 pci_write_word(arg.recv->dev->dev, ctrl_addr, step_cmd);
265 }
266 }
267 msleep(arg.recv->dwell_time * 1000);
268
269 for (int i = 0; i < arg.lanes_n; i++)
270 {
271 if (!failed_lanes[i])
272 {
273 int status_addr = LMR_LANE_STATUS(arg.recv->dev->lmr_cap_addr, arg.results[i].lane);
274 lane_status = pci_read_word(arg.recv->dev->dev, status_addr);
275 u8 step_status = GET_REG_MASK(lane_status, LMR_PLD_MARGIN_STS);
276 if (!(GET_REG_MASK(lane_status, LMR_CMD_TYPE) == marg_type
277 && GET_REG_MASK(lane_status, LMR_CMD_RECVN) == arg.recv->recvn
278 && step_status == 2
279 && GET_REG_MASK(lane_status, LMR_PLD_ERR_CNT) <= arg.recv->error_limit
280 && margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND)))
281 {
282 alive_lanes--;
283 failed_lanes[i] = true;
284 arg.results[i].steps[arg.dir] = steps_done - 1;
285 arg.results[i].statuses[arg.dir]
286 = (step_status == 3 || step_status == 1 ? MARGIN_NAK : MARGIN_LIM);
287 }
288 }
289 }
290
291 arg.steps_lane_done = steps_done;
292 margin_log_margining(arg);
293 }
294
295 for (int i = 0; i < arg.lanes_n; i++)
296 {
297 margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
298 margin_set_cmd(arg.recv->dev, arg.results[i].lane, CLEAR_ERROR_LOG(arg.recv->recvn));
299 margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
300 margin_set_cmd(arg.recv->dev, arg.results[i].lane, GO_TO_NORMAL_SETTINGS(arg.recv->recvn));
301 margin_set_cmd(arg.recv->dev, arg.results[i].lane, NO_COMMAND);
302 }
303 }
304
305 /* Awaits that Receiver is prepared through prep_dev function */
306 static bool
margin_test_receiver(struct margin_dev * dev,u8 recvn,struct margin_link_args * args,struct margin_results * results)307 margin_test_receiver(struct margin_dev *dev, u8 recvn, struct margin_link_args *args,
308 struct margin_results *results)
309 {
310 u8 *lanes_to_margin = args->lanes;
311 u8 lanes_n = args->lanes_n;
312
313 struct margin_params params;
314 struct margin_recv recv = { .dev = dev,
315 .recvn = recvn,
316 .lane_reversal = false,
317 .params = ¶ms,
318 .parallel_lanes = args->parallel_lanes ? args->parallel_lanes : 1,
319 .error_limit = args->common->error_limit,
320 .dwell_time = args->common->dwell_time };
321
322 results->recvn = recvn;
323 results->lanes_n = lanes_n;
324 margin_log_recvn(&recv);
325
326 if (!margin_check_ready_bit(dev->dev))
327 {
328 margin_log("\nMargining Ready bit is Clear.\n");
329 results->test_status = MARGIN_TEST_READY_BIT;
330 return false;
331 }
332
333 if (!read_params_internal(dev, recvn, recv.lane_reversal, ¶ms))
334 {
335 recv.lane_reversal = true;
336 if (!read_params_internal(dev, recvn, recv.lane_reversal, ¶ms))
337 {
338 margin_log("\nError during caps reading.\n");
339 results->test_status = MARGIN_TEST_CAPS;
340 return false;
341 }
342 }
343
344 results->params = params;
345
346 if (recv.parallel_lanes > params.max_lanes + 1)
347 recv.parallel_lanes = params.max_lanes + 1;
348 margin_apply_hw_quirks(&recv, args);
349 margin_log_hw_quirks(&recv);
350
351 results->tim_off_reported = params.timing_offset != 0;
352 results->volt_off_reported = params.volt_offset != 0;
353 double tim_offset = results->tim_off_reported ? (double)params.timing_offset : 50.0;
354 double volt_offset = results->volt_off_reported ? (double)params.volt_offset : 50.0;
355
356 results->tim_coef = tim_offset / (double)params.timing_steps;
357 results->volt_coef = volt_offset / (double)params.volt_steps * 10.0;
358
359 results->lane_reversal = recv.lane_reversal;
360 results->link_speed = dev->link_speed;
361 results->test_status = MARGIN_TEST_OK;
362
363 margin_log_receiver(&recv);
364
365 results->lanes = xmalloc(sizeof(struct margin_res_lane) * lanes_n);
366 for (int i = 0; i < lanes_n; i++)
367 {
368 results->lanes[i].lane
369 = recv.lane_reversal ? dev->max_width - lanes_to_margin[i] - 1 : lanes_to_margin[i];
370 }
371
372 if (args->common->run_margin)
373 {
374 if (args->common->verbosity > 0)
375 margin_log("\n");
376 struct margin_lanes_data lanes_data = { .recv = &recv,
377 .verbosity = args->common->verbosity,
378 .steps_utility = &args->common->steps_utility };
379
380 enum margin_dir dir[] = { TIM_LEFT, TIM_RIGHT, VOLT_UP, VOLT_DOWN };
381
382 u8 lanes_done = 0;
383 u8 use_lanes = 0;
384 u8 steps_t = args->steps_t ? args->steps_t : params.timing_steps;
385 u8 steps_v = args->steps_v ? args->steps_v : params.volt_steps;
386
387 while (lanes_done != lanes_n)
388 {
389 use_lanes = (lanes_done + recv.parallel_lanes > lanes_n) ? lanes_n - lanes_done :
390 recv.parallel_lanes;
391 lanes_data.lanes_numbers = lanes_to_margin + lanes_done;
392 lanes_data.lanes_n = use_lanes;
393 lanes_data.results = results->lanes + lanes_done;
394
395 for (int i = 0; i < 4; i++)
396 {
397 bool timing = dir[i] == TIM_LEFT || dir[i] == TIM_RIGHT;
398 if (!timing && !params.volt_support)
399 continue;
400 if (dir[i] == TIM_RIGHT && !params.ind_left_right_tim)
401 continue;
402 if (dir[i] == VOLT_DOWN && !params.ind_up_down_volt)
403 continue;
404
405 lanes_data.ind = timing ? params.ind_left_right_tim : params.ind_up_down_volt;
406 lanes_data.dir = dir[i];
407 lanes_data.steps_lane_total = timing ? steps_t : steps_v;
408 if (args->common->steps_utility >= lanes_data.steps_lane_total)
409 args->common->steps_utility -= lanes_data.steps_lane_total;
410 else
411 args->common->steps_utility = 0;
412 margin_test_lanes(lanes_data);
413 }
414 lanes_done += use_lanes;
415 }
416 if (args->common->verbosity > 0)
417 margin_log("\n");
418 if (recv.lane_reversal)
419 {
420 for (int i = 0; i < lanes_n; i++)
421 results->lanes[i].lane = lanes_to_margin[i];
422 }
423 }
424
425 return true;
426 }
427
428 bool
margin_read_params(struct pci_access * pacc,struct pci_dev * dev,u8 recvn,struct margin_params * params)429 margin_read_params(struct pci_access *pacc, struct pci_dev *dev, u8 recvn,
430 struct margin_params *params)
431 {
432 struct pci_cap *cap = pci_find_cap(dev, PCI_CAP_ID_EXP, PCI_CAP_NORMAL);
433 if (!cap)
434 return false;
435
436 bool dev_down = margin_port_is_down(dev);
437
438 if (recvn == 0)
439 {
440 if (dev_down)
441 recvn = 1;
442 else
443 recvn = 6;
444 }
445
446 if (recvn > 6)
447 return false;
448 if (dev_down && recvn == 6)
449 return false;
450 if (!dev_down && recvn != 6)
451 return false;
452
453 struct pci_dev *down = NULL;
454 struct pci_dev *up = NULL;
455 struct margin_link link;
456
457 if (!margin_find_pair(pacc, dev, &down, &up))
458 return false;
459
460 if (!margin_fill_link(down, up, &link))
461 return false;
462
463 struct margin_dev *dut = (dev_down ? &link.down_port : &link.up_port);
464 if (!margin_check_ready_bit(dut->dev))
465 return false;
466
467 if (!margin_prep_link(&link))
468 return false;
469
470 bool status;
471 bool lane_reversal = false;
472 status = read_params_internal(dut, recvn, lane_reversal, params);
473 if (!status)
474 {
475 lane_reversal = true;
476 status = read_params_internal(dut, recvn, lane_reversal, params);
477 }
478
479 margin_restore_link(&link);
480
481 return status;
482 }
483
484 enum margin_test_status
margin_process_args(struct margin_link * link)485 margin_process_args(struct margin_link *link)
486 {
487 struct margin_dev *dev = &link->down_port;
488 struct margin_link_args *args = &link->args;
489
490 u8 receivers_n = 2 + 2 * dev->retimers_n;
491
492 if (!args->recvs_n)
493 {
494 for (int i = 1; i < receivers_n; i++)
495 args->recvs[i - 1] = i;
496 args->recvs[receivers_n - 1] = 6;
497 args->recvs_n = receivers_n;
498 }
499 else
500 {
501 for (int i = 0; i < args->recvs_n; i++)
502 {
503 u8 recvn = args->recvs[i];
504 if (recvn < 1 || recvn > 6 || (recvn != 6 && recvn > receivers_n - 1))
505 {
506 return MARGIN_TEST_ARGS_RECVS;
507 }
508 }
509 }
510
511 if (!args->lanes_n)
512 {
513 args->lanes_n = dev->neg_width;
514 for (int i = 0; i < args->lanes_n; i++)
515 args->lanes[i] = i;
516 }
517 else
518 {
519 for (int i = 0; i < args->lanes_n; i++)
520 {
521 if (args->lanes[i] >= dev->neg_width)
522 {
523 return MARGIN_TEST_ARGS_LANES;
524 }
525 }
526 }
527
528 return MARGIN_TEST_OK;
529 }
530
531 struct margin_results *
margin_test_link(struct margin_link * link,u8 * recvs_n)532 margin_test_link(struct margin_link *link, u8 *recvs_n)
533 {
534 struct margin_link_args *args = &link->args;
535
536 bool status = margin_prep_link(link);
537
538 u8 receivers_n = status ? args->recvs_n : 1;
539 u8 *receivers = args->recvs;
540
541 margin_log_link(link);
542
543 struct margin_results *results = xmalloc(sizeof(*results) * receivers_n);
544
545 if (!status)
546 {
547 results[0].test_status = MARGIN_TEST_ASPM;
548 margin_log("\nCouldn't disable ASPM on the given Link.\n");
549 }
550
551 if (status)
552 {
553 struct margin_dev *dut;
554 for (int i = 0; i < receivers_n; i++)
555 {
556 dut = receivers[i] == 6 ? &link->up_port : &link->down_port;
557 margin_test_receiver(dut, receivers[i], args, &results[i]);
558 }
559
560 margin_restore_link(link);
561 }
562
563 *recvs_n = receivers_n;
564 return results;
565 }
566
567 void
margin_free_results(struct margin_results * results,u8 results_n)568 margin_free_results(struct margin_results *results, u8 results_n)
569 {
570 for (int i = 0; i < results_n; i++)
571 {
572 if (results[i].test_status == MARGIN_TEST_OK)
573 free(results[i].lanes);
574 }
575 free(results);
576 }
577