1 /* SPDX-License-Identifier: GPL-2.0-only */
2
3 #include <assert.h>
4 #include <console/console.h>
5 #include <delay.h>
6 #include <device/mmio.h>
7 #include <soc/emi.h>
8 #include <soc/dramc_register.h>
9 #include <soc/dramc_param.h>
10 #include <soc/dramc_pi_api.h>
11 #include <soc/spm.h>
12 #include <timer.h>
13
14 enum {
15 RX_VREF_BEGIN = 0,
16 RX_VREF_END = 31,
17 RX_VREF_STEP = 1,
18 TX_VREF_BEGIN = 0,
19 TX_VREF_END = 50,
20 TX_VREF_STEP = 2,
21 };
22
23 enum {
24 FIRST_DQ_DELAY = 0,
25 FIRST_DQS_DELAY = -48,
26 MAX_DQDLY_TAPS = 16,
27 MAX_RX_DQDLY_TAPS = 63,
28 };
29
30 #define WRITE_LEVELING_MOVD_DQS 1
31 #define TEST2_1_CAL 0x55000000
32 #define TEST2_2_CAL 0xaa000400
33
34 enum CAL_TYPE {
35 RX_WIN_RD_DQC = 0,
36 RX_WIN_TEST_ENG,
37 TX_WIN_DQ_ONLY,
38 TX_WIN_DQ_DQM,
39 };
40
41 enum RX_TYPE {
42 RX_DQ = 0,
43 RX_DQM,
44 RX_DQS,
45 };
46
47 struct win_perbit_dly {
48 s16 first_pass;
49 s16 last_pass;
50 s16 best_first;
51 s16 best_last;
52 s16 best_dqdly;
53 s16 win_center;
54 };
55
56 struct vref_perbit_dly {
57 u8 best_vref;
58 u16 max_win_sum;
59 struct win_perbit_dly perbit_dly[DQ_DATA_WIDTH];
60 };
61
62 struct tx_dly_tune {
63 u8 fine_tune;
64 u8 coarse_tune_large;
65 u8 coarse_tune_small;
66 u8 coarse_tune_large_oen;
67 u8 coarse_tune_small_oen;
68 };
69
70 struct per_byte_dly {
71 u16 max_center;
72 u16 min_center;
73 u16 final_dly;
74 };
75
76 static const u8 lp4_ca_mapping_pop[CHANNEL_MAX][CA_NUM_LP4] = {
77 [CHANNEL_A] = {1, 4, 3, 2, 0, 5},
78 [CHANNEL_B] = {0, 3, 2, 4, 1, 5},
79 };
80
dramc_auto_refresh_switch(u8 chn,bool option)81 static void dramc_auto_refresh_switch(u8 chn, bool option)
82 {
83 SET32_BITFIELDS(&ch[chn].ao.refctrl0, REFCTRL0_REFDIS, option ? 0 : 1);
84
85 if (!option) {
86 /*
87 * Because HW will actually disable autorefresh after
88 * refresh_queue empty, we need to wait until queue empty.
89 */
90 udelay(READ32_BITFIELD(&ch[chn].nao.misc_statusa,
91 MISC_STATUSA_REFRESH_QUEUE_CNT) * 4);
92 }
93 }
94
dramc_mode_reg_read(u8 chn,u8 mr_idx)95 static u16 dramc_mode_reg_read(u8 chn, u8 mr_idx)
96 {
97 u16 value;
98 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSMA, mr_idx);
99 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_MRREN, 1);
100
101 /* Wait until MRW command fired */
102 while (READ32_BITFIELD(&ch[chn].nao.spcmdresp, SPCMDRESP_MRR_RESPONSE)
103 == 0)
104 udelay(1);
105
106 value = READ32_BITFIELD(&ch[chn].nao.mrr_status, MRR_STATUS_MRR_REG);
107
108 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_MRREN, 0);
109 dramc_dbg("Read MR%d =%#x\n", mr_idx, value);
110
111 return value;
112 }
113
dramc_mode_reg_write(u8 chn,u8 mr_idx,u8 value)114 void dramc_mode_reg_write(u8 chn, u8 mr_idx, u8 value)
115 {
116 u32 ckectrl_bak = read32(&ch[chn].ao.ckectrl);
117
118 dramc_cke_fix_onoff(CKE_FIXON, chn);
119 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSMA, mr_idx);
120 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSOP, value);
121 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_MRWEN, 1);
122
123 /* Wait MRW command fired */
124 while (READ32_BITFIELD(&ch[chn].nao.spcmdresp, SPCMDRESP_MRW_RESPONSE)
125 == 0)
126 udelay(1);
127
128 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_MRWEN, 0);
129 write32(&ch[chn].ao.ckectrl, ckectrl_bak);
130 dramc_dbg("Write MR%d =0x%x\n", mr_idx, value);
131 }
132
dramc_mode_reg_read_by_rank(u8 chn,u8 rank,u8 mr_idx)133 static u8 dramc_mode_reg_read_by_rank(u8 chn, u8 rank, u8 mr_idx)
134 {
135 u8 value;
136 u32 rk_bak = READ32_BITFIELD(&ch[chn].ao.mrs, MRS_MRRRK);
137
138 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRRRK, rank);
139 value = dramc_mode_reg_read(chn, mr_idx);
140 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRRRK, rk_bak);
141
142 dramc_dbg("Mode reg read rank%d MR%d = %#x\n", rank, mr_idx, value);
143 return value;
144 }
145
dramc_mode_reg_write_by_rank(u8 chn,u8 rank,u8 mr_idx,u8 value)146 static void dramc_mode_reg_write_by_rank(u8 chn, u8 rank,
147 u8 mr_idx, u8 value)
148 {
149 u32 mrs_bak = READ32_BITFIELD(&ch[chn].ao.mrs, MRS_MRSRK);
150 dramc_dbg("Mode reg write rank%d MR%d = 0x%x\n", rank, mr_idx, value);
151
152 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, rank);
153 dramc_mode_reg_write(chn, mr_idx, value);
154 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, mrs_bak);
155 }
156
move_dramc_delay(u32 * reg_0,u32 * reg_1,u8 shift,s8 shift_coarse_tune)157 static void move_dramc_delay(u32 *reg_0, u32 *reg_1, u8 shift, s8 shift_coarse_tune)
158 {
159 s32 sum;
160 u32 tmp_0p5t, tmp_2t;
161
162 tmp_0p5t = ((read32(reg_0) >> shift) & DQ_DIV_MASK) &
163 ~(1 << DQ_DIV_SHIFT);
164 tmp_2t = (read32(reg_1) >> shift) & DQ_DIV_MASK;
165
166 sum = (tmp_2t << DQ_DIV_SHIFT) + tmp_0p5t + shift_coarse_tune;
167
168 if (sum < 0) {
169 tmp_0p5t = 0;
170 tmp_2t = 0;
171 } else {
172 tmp_2t = sum >> DQ_DIV_SHIFT;
173 tmp_0p5t = sum - (tmp_2t << DQ_DIV_SHIFT);
174 }
175
176 clrsetbits32(reg_0, DQ_DIV_MASK << shift, tmp_0p5t << shift);
177 clrsetbits32(reg_1, DQ_DIV_MASK << shift, tmp_2t << shift);
178 }
179
move_dramc_tx_dqs(u8 chn,u8 byte,s8 shift_coarse_tune)180 static void move_dramc_tx_dqs(u8 chn, u8 byte, s8 shift_coarse_tune)
181 {
182 move_dramc_delay(&ch[chn].ao.shu[0].selph_dqs1,
183 &ch[chn].ao.shu[0].selph_dqs0, byte * 4, shift_coarse_tune);
184 }
185
move_dramc_tx_dqs_oen(u8 chn,u8 byte,s8 shift_coarse_tune)186 static void move_dramc_tx_dqs_oen(u8 chn, u8 byte,
187 s8 shift_coarse_tune)
188 {
189 move_dramc_delay(&ch[chn].ao.shu[0].selph_dqs1,
190 &ch[chn].ao.shu[0].selph_dqs0, byte * 4 + OEN_SHIFT, shift_coarse_tune);
191 }
192
move_dramc_tx_dq(u8 chn,u8 rank,u8 byte,s8 shift_coarse_tune)193 static void move_dramc_tx_dq(u8 chn, u8 rank, u8 byte, s8 shift_coarse_tune)
194 {
195 /* DQM0 */
196 move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[3],
197 &ch[chn].ao.shu[0].rk[rank].selph_dq[1], byte * 4, shift_coarse_tune);
198
199 /* DQ0 */
200 move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[2],
201 &ch[chn].ao.shu[0].rk[rank].selph_dq[0], byte * 4, shift_coarse_tune);
202 }
203
move_dramc_tx_dq_oen(u8 chn,u8 rank,u8 byte,s8 shift_coarse_tune)204 static void move_dramc_tx_dq_oen(u8 chn, u8 rank,
205 u8 byte, s8 shift_coarse_tune)
206 {
207 /* DQM_OEN_0 */
208 move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[3],
209 &ch[chn].ao.shu[0].rk[rank].selph_dq[1],
210 byte * 4 + OEN_SHIFT, shift_coarse_tune);
211
212 /* DQ_OEN_0 */
213 move_dramc_delay(&ch[chn].ao.shu[0].rk[rank].selph_dq[2],
214 &ch[chn].ao.shu[0].rk[rank].selph_dq[0],
215 byte * 4 + OEN_SHIFT, shift_coarse_tune);
216 }
217
write_leveling_move_dqs_instead_of_clk(u8 chn)218 static void write_leveling_move_dqs_instead_of_clk(u8 chn)
219 {
220 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
221 move_dramc_tx_dqs(chn, byte, -WRITE_LEVELING_MOVD_DQS);
222 move_dramc_tx_dqs_oen(chn, byte, -WRITE_LEVELING_MOVD_DQS);
223
224 for (u8 rk = RANK_0; rk < RANK_MAX; rk++) {
225 move_dramc_tx_dq(chn, rk, byte, -WRITE_LEVELING_MOVD_DQS);
226 move_dramc_tx_dq_oen(chn, rk, byte, -WRITE_LEVELING_MOVD_DQS);
227 }
228 }
229 }
230
dramc_write_leveling(u8 chn,u8 rank,u8 freq_group,const u8 wr_level[CHANNEL_MAX][RANK_MAX][DQS_NUMBER])231 static void dramc_write_leveling(u8 chn, u8 rank, u8 freq_group,
232 const u8 wr_level[CHANNEL_MAX][RANK_MAX][DQS_NUMBER])
233 {
234 dramc_auto_refresh_switch(chn, false);
235
236 if (rank == RANK_0)
237 write_leveling_move_dqs_instead_of_clk(chn);
238
239 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].ca_cmd[9],
240 SHU1_R0_CA_CMD9_RG_RK0_ARPI_CLK, 0);
241
242 for (size_t byte = 0; byte < DQS_NUMBER; byte++) {
243 u32 wrlevel_dq_delay = wr_level[chn][rank][byte] + 0x10;
244 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7],
245 FINE_TUNE_PBYTE, wr_level[chn][rank][byte]);
246 if (wrlevel_dq_delay >= 0x40) {
247 wrlevel_dq_delay -= 0x40;
248 move_dramc_tx_dq(chn, rank, byte, 2);
249 move_dramc_tx_dq_oen(chn, rank, byte, 2);
250 }
251 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7],
252 FINE_TUNE_DQM, wrlevel_dq_delay,
253 FINE_TUNE_DQ, wrlevel_dq_delay);
254 }
255 }
256
cbt_set_perbit_delay_cell(u8 chn,u8 rank)257 static void cbt_set_perbit_delay_cell(u8 chn, u8 rank)
258 {
259 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].ca_cmd[0],
260 SHU1_R0_CA_CMD0_RK0_TX_ARCA0_DLY, 0,
261 SHU1_R0_CA_CMD0_RK0_TX_ARCA1_DLY, 0,
262 SHU1_R0_CA_CMD0_RK0_TX_ARCA2_DLY, 0,
263 SHU1_R0_CA_CMD0_RK0_TX_ARCA3_DLY, 0,
264 SHU1_R0_CA_CMD0_RK0_TX_ARCA4_DLY, 0,
265 SHU1_R0_CA_CMD0_RK0_TX_ARCA5_DLY, 0);
266 }
267
set_dram_mr_cbt_on_off(u8 chn,u8 rank,u8 fsp,bool cbt_on,struct mr_value * mr,u32 cbt_mode)268 static void set_dram_mr_cbt_on_off(u8 chn, u8 rank, u8 fsp,
269 bool cbt_on, struct mr_value *mr, u32 cbt_mode)
270 {
271 u8 MR13Value = mr->MR13Value;
272
273 if (cbt_on) {
274 MR13Value |= 0x1;
275 if (fsp == FSP_1)
276 MR13Value &= 0x7f;
277 else
278 MR13Value |= 0x80;
279
280 if (cbt_mode)
281 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_BYTEMODECBTEN, 1);
282 } else {
283 MR13Value &= 0xfe;
284 if (fsp == FSP_1)
285 MR13Value |= 0x80;
286 else
287 MR13Value &= 0x7f;
288 }
289
290 dramc_mode_reg_write_by_rank(chn, rank, 13, MR13Value);
291 mr->MR13Value = MR13Value;
292 }
293
cbt_set_fsp(u8 chn,u8 rank,u8 fsp,struct mr_value * mr)294 static void cbt_set_fsp(u8 chn, u8 rank, u8 fsp, struct mr_value *mr)
295 {
296 u8 MR13Value = mr->MR13Value;
297
298 if (fsp == FSP_0) {
299 MR13Value &= ~(BIT(6));
300 MR13Value &= 0x7f;
301 } else {
302 MR13Value |= BIT(6);
303 MR13Value |= 0x80;
304 }
305
306 dramc_mode_reg_write_by_rank(chn, rank, 13, MR13Value);
307 mr->MR13Value = MR13Value;
308 }
309
o1_path_on_off(u8 cbt_on)310 static void o1_path_on_off(u8 cbt_on)
311 {
312 u8 fix_dqien = (cbt_on == 1) ? 3 : 0;
313
314 for (u8 chn = 0; chn < CHANNEL_MAX; chn++) {
315 SET32_BITFIELDS(&ch[chn].ao.padctrl, PADCTRL_FIXDQIEN, fix_dqien);
316 SET32_BITFIELDS(&ch[chn].phy.b[0].dq[5],
317 B0_DQ5_RG_RX_ARDQ_EYE_VREF_EN_B0, cbt_on);
318 SET32_BITFIELDS(&ch[chn].phy.b[1].dq[5],
319 B1_DQ5_RG_RX_ARDQ_EYE_VREF_EN_B1, cbt_on);
320 SET32_BITFIELDS(&ch[chn].phy.b[0].dq[3],
321 B0_DQ3_RG_RX_ARDQ_SMT_EN_B0, cbt_on);
322 SET32_BITFIELDS(&ch[chn].phy.b[1].dq[3],
323 B1_DQ3_RG_RX_ARDQ_SMT_EN_B1, cbt_on);
324 }
325 udelay(1);
326 }
327
cbt_entry(u8 chn,u8 rank,u8 fsp,struct mr_value * mr,u32 cbt_mode)328 static void cbt_entry(u8 chn, u8 rank, u8 fsp, struct mr_value *mr, u32 cbt_mode)
329 {
330 SET32_BITFIELDS(&ch[chn].ao.dramc_pd_ctrl,
331 DRAMC_PD_CTRL_PHYCLKDYNGEN, 0,
332 DRAMC_PD_CTRL_DCMEN, 0);
333 SET32_BITFIELDS(&ch[chn].ao.stbcal, STBCAL_DQSIENCG_NORMAL_EN, 0);
334 SET32_BITFIELDS(&ch[chn].ao.dramc_pd_ctrl, DRAMC_PD_CTRL_MIOCKCTRLOFF, 1);
335
336 dramc_cke_fix_onoff(CKE_FIXON, chn);
337 set_dram_mr_cbt_on_off(chn, rank, fsp, true, mr, cbt_mode);
338
339 if (cbt_mode == 0)
340 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_WRITE_LEVEL_EN, 1);
341
342 udelay(1);
343 dramc_cke_fix_onoff(CKE_FIXOFF, chn);
344 o1_path_on_off(1);
345 }
346
cbt_exit(u8 chn,u8 rank,u8 fsp,struct mr_value * mr,u32 cbt_mode)347 static void cbt_exit(u8 chn, u8 rank, u8 fsp, struct mr_value *mr, u32 cbt_mode)
348 {
349 dramc_cke_fix_onoff(CKE_FIXON, chn);
350
351 udelay(1);
352 set_dram_mr_cbt_on_off(chn, rank, fsp, false, mr, cbt_mode);
353 o1_path_on_off(0);
354
355 if (cbt_mode)
356 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_BYTEMODECBTEN, 0);
357 }
358
cbt_set_vref(u8 chn,u8 rank,u8 vref,bool is_final,u32 cbt_mode)359 static void cbt_set_vref(u8 chn, u8 rank, u8 vref, bool is_final, u32 cbt_mode)
360 {
361 if (cbt_mode == 0 && !is_final) {
362 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_DMVREFCA, vref);
363 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_DQS_SEL, 1);
364 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_DQSBX_G, 0xa);
365 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_DQS_WLEV, 1);
366 udelay(1);
367 SET32_BITFIELDS(&ch[chn].ao.write_lev, WRITE_LEV_DQS_WLEV, 0);
368 } else {
369 vref |= BIT(6);
370 dramc_dbg("final_vref: %#x\n", vref);
371
372 /* CBT set vref */
373 dramc_mode_reg_write_by_rank(chn, rank, 12, vref);
374 }
375 }
376
cbt_set_ca_clk_result(u8 chn,u8 rank,const struct sdram_params * params)377 static void cbt_set_ca_clk_result(u8 chn, u8 rank,
378 const struct sdram_params *params)
379 {
380 const u8 *perbit_dly;
381 u8 clk_dly = params->cbt_clk_dly[chn][rank];
382 u8 cmd_dly = params->cbt_cmd_dly[chn][rank];
383 const u8 *ca_mapping = lp4_ca_mapping_pop[chn];
384
385 for (u8 rk = 0; rk < rank + 1; rk++) {
386 /* Set CLK and CA delay */
387 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rk].ca_cmd[9],
388 SHU1_R0_CA_CMD9_RG_RK0_ARPI_CMD, cmd_dly,
389 SHU1_R0_CA_CMD9_RG_RK0_ARPI_CLK, clk_dly);
390 udelay(1);
391
392 perbit_dly = params->cbt_ca_perbit_delay[chn][rk];
393
394 /* Set CA perbit delay line calibration results */
395 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rk].ca_cmd[0],
396 SHU1_R0_CA_CMD0_RK0_TX_ARCA0_DLY, perbit_dly[ca_mapping[0]],
397 SHU1_R0_CA_CMD0_RK0_TX_ARCA1_DLY, perbit_dly[ca_mapping[1]],
398 SHU1_R0_CA_CMD0_RK0_TX_ARCA2_DLY, perbit_dly[ca_mapping[2]],
399 SHU1_R0_CA_CMD0_RK0_TX_ARCA3_DLY, perbit_dly[ca_mapping[3]],
400 SHU1_R0_CA_CMD0_RK0_TX_ARCA4_DLY, perbit_dly[ca_mapping[4]],
401 SHU1_R0_CA_CMD0_RK0_TX_ARCA5_DLY, perbit_dly[ca_mapping[5]]);
402 }
403 }
404
get_cbt_vref_pinmux_value(u8 chn,u8 vref_level,u32 cbt_mode)405 static u8 get_cbt_vref_pinmux_value(u8 chn, u8 vref_level, u32 cbt_mode)
406 {
407 u8 vref_bit, vref_new, vref_org;
408
409 vref_new = 0;
410 vref_org = BIT(6) | (vref_level & 0x3f);
411
412 if (cbt_mode) {
413 dramc_dbg("vref_org: %#x for byte mode\n", vref_org);
414
415 return vref_org;
416 }
417 for (vref_bit = 0; vref_bit < 8; vref_bit++) {
418 if (vref_org & (1 << vref_bit))
419 vref_new |= (1 << phy_mapping[chn][vref_bit]);
420 }
421
422 dramc_dbg("vref_new: %#x --> %#x\n", vref_org, vref_new);
423
424 return vref_new;
425 }
426
cbt_dramc_dfs_direct_jump(u8 shu_level,bool run_dvfs)427 static void cbt_dramc_dfs_direct_jump(u8 shu_level, bool run_dvfs)
428 {
429 u8 shu_ack = 0;
430 static bool phy_pll_en = true;
431
432 if (!run_dvfs)
433 return;
434
435 for (u8 chn = 0; chn < CHANNEL_MAX; chn++)
436 shu_ack |= (0x1 << chn);
437
438 if (phy_pll_en) {
439 dramc_dbg("Disable CLRPLL\n");
440 SET32_BITFIELDS(&ch[0].phy.pll2, PLL2_RG_RCLRPLL_EN, 0);
441 dramc_dbg("DFS jump to CLRPLL, shu lev=%d, ACK=%x\n",
442 shu_level, shu_ack);
443 } else {
444 dramc_dbg("Disable PHYPLL\n");
445 SET32_BITFIELDS(&ch[0].phy.pll1, PLL1_RG_RPHYPLL_EN, 0);
446 dramc_dbg("DFS jump to PHYPLL, shu lev=%d, ACK=%x\n",
447 shu_level, shu_ack);
448 }
449
450 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
451 SPM_POWER_ON_VAL0_SC_PHYPLL1_SHU_EN_PCM, 0);
452 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
453 SPM_POWER_ON_VAL0_SC_PHYPLL2_SHU_EN_PCM, 0);
454 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
455 SPM_POWER_ON_VAL0_SC_DR_SHU_LEVEL_PCM, 0);
456 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
457 SPM_POWER_ON_VAL0_SC_DR_SHU_LEVEL_PCM, shu_level);
458
459 if (phy_pll_en) {
460 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
461 SPM_POWER_ON_VAL0_SC_PHYPLL2_SHU_EN_PCM, 1);
462 udelay(1);
463 SET32_BITFIELDS(&ch[0].phy.pll2, PLL2_RG_RCLRPLL_EN, 1);
464 dramc_dbg("Enable CLRPLL\n");
465 } else {
466 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
467 SPM_POWER_ON_VAL0_SC_PHYPLL1_SHU_EN_PCM, 1);
468 udelay(1);
469 SET32_BITFIELDS(&ch[0].phy.pll1, PLL1_RG_RPHYPLL_EN, 1);
470 dramc_dbg("Enable PHYPLL\n");
471 }
472
473 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
474 SPM_POWER_ON_VAL0_SC_TX_TRACKING_DIS, 3);
475
476 udelay(20);
477 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
478 SPM_POWER_ON_VAL0_SC_DDRPHY_FB_CK_EN_PCM, 1);
479 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
480 SPM_POWER_ON_VAL0_SC_DPHY_RXDLY_TRACK_EN, 0);
481 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
482 SPM_POWER_ON_VAL0_SC_DR_SHU_EN_PCM, 1);
483
484 while ((READ32_BITFIELD(&mtk_spm->dramc_dpy_clk_sw_con,
485 DRAMC_DPY_CLK_SW_CON_SC_DMDRAMCSHU_ACK) & shu_ack)
486 != shu_ack) {
487 dramc_dbg("wait shu_en ack.\n");
488 }
489 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
490 SPM_POWER_ON_VAL0_SC_DR_SHU_EN_PCM, 0);
491
492 if (shu_level == 0)
493 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
494 SPM_POWER_ON_VAL0_SC_DPHY_RXDLY_TRACK_EN, 3);
495
496 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
497 SPM_POWER_ON_VAL0_SC_TX_TRACKING_DIS, 0);
498 SET32_BITFIELDS(&mtk_spm->spm_power_on_val0,
499 SPM_POWER_ON_VAL0_SC_DDRPHY_FB_CK_EN_PCM, 0);
500
501 if (phy_pll_en)
502 SET32_BITFIELDS(&ch[0].phy.pll1, PLL1_RG_RPHYPLL_EN, 0);
503 else
504 SET32_BITFIELDS(&ch[0].phy.pll2, PLL2_RG_RCLRPLL_EN, 0);
505 dramc_dbg("Shuffle flow complete\n");
506
507 phy_pll_en = !phy_pll_en;
508 }
509
cbt_switch_freq(cbt_freq freq,bool run_dvfs)510 static void cbt_switch_freq(cbt_freq freq, bool run_dvfs)
511 {
512 if (freq == CBT_LOW_FREQ)
513 cbt_dramc_dfs_direct_jump(DRAM_DFS_SHUFFLE_MAX - 1, run_dvfs);
514 else
515 cbt_dramc_dfs_direct_jump(DRAM_DFS_SHUFFLE_1, run_dvfs);
516 }
517
dramc_cmd_bus_training(u8 chn,u8 rank,u8 freq_group,const struct sdram_params * params,struct mr_value * mr,bool run_dvfs)518 static void dramc_cmd_bus_training(u8 chn, u8 rank, u8 freq_group,
519 const struct sdram_params *params, struct mr_value *mr,
520 bool run_dvfs)
521 {
522 u8 final_vref, cs_dly;
523 u8 fsp = get_freq_fsq(freq_group);
524 u32 cbt_mode = params->cbt_mode_extern;
525
526 cs_dly = params->cbt_cs_dly[chn][rank];
527 final_vref = params->cbt_final_vref[chn][rank];
528
529 struct reg_value regs_bak[] = {
530 {&ch[chn].ao.dramc_pd_ctrl},
531 {&ch[chn].ao.stbcal},
532 {&ch[chn].ao.ckectrl},
533 {&ch[chn].ao.write_lev},
534 {&ch[chn].ao.refctrl0},
535 {&ch[chn].ao.spcmdctrl},
536 };
537
538 for (int i = 0; i < ARRAY_SIZE(regs_bak); i++)
539 regs_bak[i].value = read32(regs_bak[i].addr);
540
541 dramc_auto_refresh_switch(chn, false);
542
543 if (rank == RANK_1) {
544 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, rank);
545 SET32_BITFIELDS(&ch[chn].ao.rkcfg, RKCFG_TXRANK, rank);
546 SET32_BITFIELDS(&ch[chn].ao.rkcfg, RKCFG_TXRANKFIX, 1);
547 SET32_BITFIELDS(&ch[chn].ao.mpc_option, MPC_OPTION_MPCRKEN, 0);
548 }
549
550 cbt_set_perbit_delay_cell(chn, rank);
551
552 if (cbt_mode == 0) {
553 cbt_mrr_pinmux_mapping();
554 if (fsp == FSP_1)
555 cbt_switch_freq(CBT_LOW_FREQ, run_dvfs);
556 cbt_entry(chn, rank, fsp, mr, cbt_mode);
557 udelay(1);
558 if (fsp == FSP_1)
559 cbt_switch_freq(CBT_HIGH_FREQ, run_dvfs);
560 }
561
562 u8 new_vref = get_cbt_vref_pinmux_value(chn, final_vref, cbt_mode);
563
564 if (cbt_mode) {
565 if (fsp == FSP_1)
566 cbt_switch_freq(CBT_LOW_FREQ, run_dvfs);
567
568 cbt_set_fsp(chn, rank, fsp, mr);
569 cbt_set_vref(chn, rank, new_vref, true, cbt_mode);
570 cbt_entry(chn, rank, fsp, mr, cbt_mode);
571 udelay(1);
572
573 if (fsp == FSP_1)
574 cbt_switch_freq(CBT_HIGH_FREQ, run_dvfs);
575 } else {
576 cbt_set_vref(chn, rank, new_vref, false, cbt_mode);
577 }
578
579 cbt_set_ca_clk_result(chn, rank, params);
580 udelay(1);
581
582 for (u8 rk = 0; rk < rank + 1; rk++) {
583 /* Set CLK and CS delay */
584 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rk].ca_cmd[9],
585 SHU1_R0_CA_CMD9_RG_RK0_ARPI_CS, cs_dly);
586 }
587
588 if (fsp == FSP_1)
589 cbt_switch_freq(CBT_LOW_FREQ, run_dvfs);
590 cbt_exit(chn, rank, fsp, mr, cbt_mode);
591
592 if (cbt_mode == 0) {
593 cbt_set_fsp(chn, rank, fsp, mr);
594 cbt_set_vref(chn, rank, final_vref, true, cbt_mode);
595 }
596
597 if (fsp == FSP_1)
598 cbt_switch_freq(CBT_HIGH_FREQ, run_dvfs);
599
600 /* restore MRR pinmux */
601 set_mrr_pinmux_mapping();
602 if (rank == RANK_1) {
603 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, 0);
604 SET32_BITFIELDS(&ch[chn].ao.rkcfg, RKCFG_TXRANK, 0);
605 SET32_BITFIELDS(&ch[chn].ao.rkcfg, RKCFG_TXRANKFIX, 0);
606 SET32_BITFIELDS(&ch[chn].ao.mpc_option, MPC_OPTION_MPCRKEN, 0x1);
607 }
608
609 for (int i = 0; i < ARRAY_SIZE(regs_bak); i++)
610 write32(regs_bak[i].addr, regs_bak[i].value);
611 }
612
dramc_read_dbi_onoff(size_t chn,bool on)613 static void dramc_read_dbi_onoff(size_t chn, bool on)
614 {
615 for (size_t b = 0; b < 2; b++)
616 SET32_BITFIELDS(&ch[chn].phy.shu[0].b[b].dq[7],
617 SHU1_B0_DQ7_R_DMDQMDBI_SHU_B0, on);
618 }
619
dramc_write_dbi_onoff(size_t chn,bool onoff)620 static void dramc_write_dbi_onoff(size_t chn, bool onoff)
621 {
622 SET32_BITFIELDS(&ch[chn].ao.shu[0].wodt, SHU1_WODT_DBIWR, onoff);
623 }
624
dramc_phy_dcm_2_channel(u8 chn,bool en)625 static void dramc_phy_dcm_2_channel(u8 chn, bool en)
626 {
627 clrsetbits32(&ch[chn].phy.misc_cg_ctrl0, (0x3 << 19) | (0x3ff << 8),
628 ((en ? 0 : 0x1) << 19) | ((en ? 0 : 0x1ff) << 9) | (1 << 8));
629
630 for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) {
631 struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i];
632 for (size_t b = 0; b < 2; b++)
633 clrsetbits32(&shu->b[b].dq[8], 0x1fff << 19,
634 ((en ? 0 : 0x7ff) << 22) | (0x1 << 21) |
635 ((en ? 0 : 0x3) << 19));
636 clrbits32(&shu->ca_cmd[8], 0x1fff << 19);
637 }
638 clrsetbits32(&ch[chn].phy.misc_cg_ctrl5, (0x7 << 16) | (0x7 << 20),
639 ((en ? 0x7 : 0) << 16) | ((en ? 0x7 : 0) << 20));
640 }
641
dramc_enable_phy_dcm(u8 chn,bool en)642 void dramc_enable_phy_dcm(u8 chn, bool en)
643 {
644 clrbits32(&ch[chn].phy.b[0].dll_fine_tune[1], 0x1 << 20);
645 clrbits32(&ch[chn].phy.b[1].dll_fine_tune[1], 0x1 << 20);
646 clrbits32(&ch[chn].phy.ca_dll_fine_tune[1], 0x1 << 20);
647
648 for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) {
649 struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i];
650 setbits32(&shu->b[0].dll[0], 0x1);
651 setbits32(&shu->b[1].dll[0], 0x1);
652 setbits32(&shu->ca_dll[0], 0x1);
653 }
654
655 clrsetbits32(&ch[chn].ao.dramc_pd_ctrl,
656 (0x1 << 0) | (0x1 << 1) | (0x1 << 2) |
657 (0x1 << 5) | (0x1 << 26) | (0x1 << 30) | (0x1 << 31),
658 ((en ? 0x1 : 0) << 0) | ((en ? 0x1 : 0) << 1) |
659 ((en ? 0x1 : 0) << 2) | ((en ? 0 : 0x1) << 5) |
660 ((en ? 0 : 0x1) << 26) | ((en ? 0x1 : 0) << 30) |
661 ((en ? 0x1 : 0) << 31));
662
663 /* DCM on: CHANNEL_EMI free run; DCM off: mem_dcm */
664 write32(&ch[chn].phy.misc_cg_ctrl2,
665 0x8060033e | (0x40 << (en ? 0x1 : 0)));
666 write32(&ch[chn].phy.misc_cg_ctrl2,
667 0x8060033f | (0x40 << (en ? 0x1 : 0)));
668 write32(&ch[chn].phy.misc_cg_ctrl2,
669 0x8060033e | (0x40 << (en ? 0x1 : 0)));
670
671 clrsetbits32(&ch[chn].phy.misc_ctrl3, 0x3 << 26,
672 (en ? 0 : 0x3) << 26);
673 for (size_t i = 0; i < DRAM_DFS_SHUFFLE_MAX; i++) {
674 u32 mask = 0x7 << 17;
675 u32 value = (en ? 0x7 : 0) << 17;
676 struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[i];
677
678 clrsetbits32(&shu->b[0].dq[7], mask, value);
679 clrsetbits32(&shu->b[1].dq[7], mask, value);
680 clrsetbits32(&shu->ca_cmd[7], mask, value);
681 }
682
683 dramc_phy_dcm_2_channel(chn, en);
684 }
685
dramc_reset_delay_chain_before_calibration(size_t chn)686 static void dramc_reset_delay_chain_before_calibration(size_t chn)
687 {
688 for (size_t rank = 0; rank < RANK_MAX; rank++) {
689 struct dramc_ddrphy_regs_shu_rk *rk =
690 &ch[chn].phy.shu[0].rk[rank];
691 clrbits32(&rk->ca_cmd[0], 0xffffff << 0);
692 clrbits32(&rk->b[0].dq[0], 0xfffffff << 0);
693 clrbits32(&rk->b[1].dq[0], 0xfffffff << 0);
694 clrbits32(&rk->b[0].dq[1], 0xf << 0);
695 clrbits32(&rk->b[1].dq[1], 0xf << 0);
696 }
697 }
698
dramc_hw_gating_onoff(u8 chn,bool on)699 void dramc_hw_gating_onoff(u8 chn, bool on)
700 {
701 clrsetbits32(&ch[chn].ao.shuctrl2, 0x3 << 14,
702 (on ? 0x3 : 0) << 14);
703 clrsetbits32(&ch[chn].ao.stbcal2, 0x1 << 28, (on ? 0x1 : 0) << 28);
704 clrsetbits32(&ch[chn].ao.stbcal, 0x1 << 24, (on ? 0x1 : 0) << 24);
705 clrsetbits32(&ch[chn].ao.stbcal, 0x1 << 22, (on ? 0x1 : 0) << 22);
706 }
707
dramc_rx_input_delay_tracking_init_by_freq(u8 chn,u8 freq_group)708 static void dramc_rx_input_delay_tracking_init_by_freq(u8 chn, u8 freq_group)
709 {
710 u8 dvs_delay;
711
712 struct ddrphy_ao_shu *shu = &ch[chn].phy.shu[0];
713
714 switch (freq_group) {
715 case LP4X_DDR1600:
716 dvs_delay = 5;
717 break;
718 case LP4X_DDR2400:
719 dvs_delay = 4;
720 break;
721 case LP4X_DDR3200:
722 case LP4X_DDR3600:
723 dvs_delay = 3;
724 break;
725 default:
726 die("Invalid DDR frequency group %u\n", freq_group);
727 return;
728 }
729
730 clrsetbits32(&shu->b[0].dq[5], 0x7 << 20, dvs_delay << 20);
731 clrsetbits32(&shu->b[1].dq[5], 0x7 << 20, dvs_delay << 20);
732 clrbits32(&shu->b[0].dq[7], (0x1 << 12) | (0x1 << 13));
733 clrbits32(&shu->b[1].dq[7], (0x1 << 12) | (0x1 << 13));
734 }
735
dramc_apply_config_before_calibration(u8 freq_group,u32 cbt_mode)736 void dramc_apply_config_before_calibration(u8 freq_group, u32 cbt_mode)
737 {
738 for (u8 chn = 0; chn < CHANNEL_MAX; chn++) {
739 dramc_enable_phy_dcm(chn, false);
740 dramc_reset_delay_chain_before_calibration(chn);
741
742 setbits32(&ch[chn].ao.shu[0].conf[3], 0x1ff << 16);
743 setbits32(&ch[chn].ao.spcmdctrl, 0x1 << 24);
744 clrsetbits32(&ch[chn].ao.shu[0].scintv, 0x1f << 1, 0x1b << 1);
745
746 for (u8 shu = DRAM_DFS_SHUFFLE_1; shu < DRAM_DFS_SHUFFLE_MAX;
747 shu++)
748 setbits32(&ch[chn].ao.shu[shu].conf[3], 0x1ff << 0);
749
750 clrbits32(&ch[chn].ao.dramctrl, 0x1 << 18);
751 clrbits32(&ch[chn].ao.spcmdctrl, 0x1 << 31);
752 clrbits32(&ch[chn].ao.spcmdctrl, 0x1 << 30);
753
754 if (cbt_mode == CBT_R0_R1_NORMAL) {
755 clrbits32(&ch[chn].ao.dqsoscr, 0x1 << 26);
756 clrbits32(&ch[chn].ao.dqsoscr, 0x1 << 25);
757 } else if (cbt_mode == CBT_R0_R1_BYTE) {
758 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 26);
759 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 25);
760 } else if (cbt_mode == CBT_R0_NORMAL_R1_BYTE) {
761 clrbits32(&ch[chn].ao.dqsoscr, 0x1 << 26);
762 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 25);
763 } else if (cbt_mode == CBT_R0_BYTE_R1_NORMAL) {
764 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 26);
765 clrbits32(&ch[chn].ao.dqsoscr, 0x1 << 25);
766 }
767
768 dramc_write_dbi_onoff(chn, false);
769 dramc_read_dbi_onoff(chn, false);
770
771 setbits32(&ch[chn].ao.spcmdctrl, 0x1 << 29);
772 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 24);
773 for (u8 shu = DRAM_DFS_SHUFFLE_1; shu < DRAM_DFS_SHUFFLE_MAX;
774 shu++)
775 setbits32(&ch[chn].ao.shu[shu].scintv, 0x1 << 30);
776
777 clrbits32(&ch[chn].ao.dummy_rd, (0x1 << 7) | (0x7 << 20));
778 dramc_hw_gating_onoff(chn, false);
779 clrbits32(&ch[chn].ao.stbcal2, 0x1 << 28);
780
781 for (size_t r = 0; r < 2; r++) {
782 for (size_t b = 0; b < 2; b++)
783 clrbits32(&ch[chn].phy.r[r].b[b].rxdvs[2],
784 (0x1 << 28) | (0x1 << 23) | (0x3 << 30));
785 clrbits32(&ch[chn].phy.r0_ca_rxdvs[2], 0x3 << 30);
786 }
787 setbits32(&ch[chn].phy.misc_ctrl1, 0x1 << 7);
788 clrbits32(&ch[chn].ao.refctrl0, 0x1 << 18);
789 clrbits32(&ch[chn].ao.mrs, 0x3 << 24);
790 setbits32(&ch[chn].ao.mpc_option, 0x1 << 17);
791 clrsetbits32(&ch[chn].phy.b[0].dq[6], 0x3 << 0, 0x1 << 0);
792 clrsetbits32(&ch[chn].phy.b[1].dq[6], 0x3 << 0, 0x1 << 0);
793 clrsetbits32(&ch[chn].phy.ca_cmd[6], 0x3 << 0, 0x1 << 0);
794
795 dramc_rx_input_delay_tracking_init_by_freq(chn, freq_group);
796
797 setbits32(&ch[chn].ao.dummy_rd, 0x1 << 25);
798 setbits32(&ch[chn].ao.drsctrl, 0x1 << 0);
799 if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600)
800 clrbits32(&ch[chn].ao.shu[0].drving[0], 0x1 << 31);
801 else
802 setbits32(&ch[chn].ao.shu[0].drving[0], 0x1 << 31);
803 }
804 }
805
dramc_set_mr13_vrcg_to_normal(u8 chn,const struct mr_value * mr,u32 rk_num)806 static void dramc_set_mr13_vrcg_to_normal(u8 chn, const struct mr_value *mr, u32 rk_num)
807 {
808 for (u8 rank = 0; rank < rk_num; rank++)
809 dramc_mode_reg_write_by_rank(chn, rank, 13,
810 mr->MR13Value & ~(0x1 << 3));
811
812 for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++)
813 clrbits32(&ch[chn].ao.shu[shu].hwset_vrcg, 0x1 << 19);
814 }
815
dramc_apply_config_after_calibration(const struct mr_value * mr,u32 rk_num)816 void dramc_apply_config_after_calibration(const struct mr_value *mr, u32 rk_num)
817 {
818 for (size_t chn = 0; chn < CHANNEL_MAX; chn++) {
819 write32(&ch[chn].phy.misc_cg_ctrl4, 0x11400000);
820 clrbits32(&ch[chn].ao.refctrl1, 0x1 << 7);
821 clrbits32(&ch[chn].ao.shuctrl, 0x1 << 2);
822 clrbits32(&ch[chn].phy.ca_cmd[6], 0x1 << 6);
823 dramc_set_mr13_vrcg_to_normal(chn, mr, rk_num);
824
825 clrbits32(&ch[chn].phy.b[0].dq[6], 0x3);
826 clrbits32(&ch[chn].phy.b[1].dq[6], 0x3);
827 clrbits32(&ch[chn].phy.ca_cmd[6], 0x3);
828 setbits32(&ch[chn].phy.b[0].dq[6], 0x1 << 5);
829 setbits32(&ch[chn].phy.b[1].dq[6], 0x1 << 5);
830 setbits32(&ch[chn].phy.ca_cmd[6], 0x1 << 5);
831
832 clrbits32(&ch[chn].ao.impcal, 0x3 << 24);
833 clrbits32(&ch[chn].phy.misc_imp_ctrl0, 0x4);
834 clrbits32(&ch[chn].phy.misc_cg_ctrl0, 0xf);
835
836 clrbits32(&ch[chn].phy.misc_ctrl0, 0x1 << 31);
837 clrbits32(&ch[chn].phy.misc_ctrl1, 0x1 << 25);
838
839 setbits32(&ch[chn].ao.spcmdctrl, 1 << 29);
840 setbits32(&ch[chn].ao.dqsoscr, 1 << 24);
841
842 for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++)
843 clrbits32(&ch[chn].ao.shu[shu].scintv, 0x1 << 30);
844
845 clrbits32(&ch[chn].ao.dummy_rd, (0x7 << 20) | (0x1 << 7));
846 dramc_cke_fix_onoff(CKE_DYNAMIC, chn);
847 clrbits32(&ch[chn].ao.dramc_pd_ctrl, 0x1 << 26);
848
849 clrbits32(&ch[chn].ao.eyescan, 0x7 << 8);
850 clrsetbits32(&ch[chn].ao.test2_4, 0x7 << 28, 0x4 << 28);
851 }
852 }
853
dramc_rx_dqs_isi_pulse_cg_switch(u8 chn,bool flag)854 static void dramc_rx_dqs_isi_pulse_cg_switch(u8 chn, bool flag)
855 {
856 for (size_t b = 0; b < 2; b++)
857 clrsetbits32(&ch[chn].phy.b[b].dq[6], 1 << 5,
858 (flag ? 1 : 0) << 5);
859 }
860
dramc_set_rank_engine2(u8 chn,u8 rank)861 static void dramc_set_rank_engine2(u8 chn, u8 rank)
862 {
863 /* Select CTO_AGENT1_RANK */
864 SET32_BITFIELDS(&ch[chn].ao.dramctrl, DRAMCTRL_ADRDECEN_TARKMODE, 1);
865 SET32_BITFIELDS(&ch[chn].ao.test2_4, TEST2_4_TESTAGENTRKSEL, 0);
866 SET32_BITFIELDS(&ch[chn].ao.test2_4, TEST2_4_TESTAGENTRK, rank);
867 }
868
dramc_engine2_setpat(u8 chn,bool test_pat)869 static void dramc_engine2_setpat(u8 chn, bool test_pat)
870 {
871 SET32_BITFIELDS(&ch[chn].ao.test2_4,
872 TEST2_4_TEST_REQ_LEN1, 0,
873 TEST2_4_TESTXTALKPAT, 0,
874 TEST2_4_TESTAUDMODE, 0,
875 TEST2_4_TESTAUDBITINV, 0);
876
877 if (!test_pat) {
878 SET32_BITFIELDS(&ch[chn].ao.perfctl0, PERFCTL0_RWOFOEN, 1);
879 SET32_BITFIELDS(&ch[chn].ao.test2_4,
880 TEST2_4_TESTSSOPAT, 0,
881 TEST2_4_TESTSSOXTALKPAT, 0,
882 TEST2_4_TESTXTALKPAT, 1);
883 } else {
884 SET32_BITFIELDS(&ch[chn].ao.test2_4,
885 TEST2_4_TESTAUDINIT, 0x11,
886 TEST2_4_TESTAUDINC, 0xd,
887 TEST2_4_TESTAUDBITINV, 1);
888 }
889 SET32_BITFIELDS(&ch[chn].ao.test2_3,
890 TEST2_3_TESTAUDPAT, test_pat, TEST2_3_TESTCNT, 0);
891 }
892
dramc_engine2_init(u8 chn,u8 rank,u32 t2_1,u32 t2_2,bool test_pat)893 static void dramc_engine2_init(u8 chn, u8 rank, u32 t2_1, u32 t2_2, bool test_pat)
894 {
895 dramc_set_rank_engine2(chn, rank);
896
897 SET32_BITFIELDS(&ch[chn].ao.dummy_rd,
898 DUMMY_RD_DQSG_DMYRD_EN, 0,
899 DUMMY_RD_DQSG_DMYWR_EN, 0,
900 DUMMY_RD_DUMMY_RD_EN, 0,
901 DUMMY_RD_SREF_DMYRD_EN, 0,
902 DUMMY_RD_DMY_RD_DBG, 0,
903 DUMMY_RD_DMY_WR_DBG, 0);
904 SET32_BITFIELDS(&ch[chn].nao.testchip_dma1,
905 TESTCHIP_DMA1_DMA_LP4MATAB_OPT, 0);
906 SET32_BITFIELDS(&ch[chn].ao.test2_3,
907 TEST2_3_TEST2W, 0,
908 TEST2_3_TEST2R, 0,
909 TEST2_3_TEST1, 0);
910 SET32_BITFIELDS(&ch[chn].ao.test2_0, TEST2_0_PAT0, t2_1 >> 24,
911 TEST2_0_PAT1, t2_2 >> 24);
912 SET32_BITFIELDS(&ch[chn].ao.test2_1, TEST2_1_TEST2_BASE,
913 t2_1 & 0xffffff);
914 SET32_BITFIELDS(&ch[chn].ao.test2_2, TEST2_2_TEST2_OFF,
915 t2_2 & 0xffffff);
916
917 dramc_engine2_setpat(chn, test_pat);
918 }
919
dramc_engine2_check_complete(u8 chn,u8 status)920 static void dramc_engine2_check_complete(u8 chn, u8 status)
921 {
922 u32 loop = 0;
923 /* In some case test engine finished but the complete signal late come,
924 * system will wait very long time. Hence, we set a timeout here.
925 * After system receive complete signal or wait until time out
926 * it will return, the caller will check compare result to verify
927 * whether engine success.
928 */
929 while (wait_us(100, read32(&ch[chn].nao.testrpt) & status) != status) {
930 if (loop++ > 100)
931 dramc_dbg("MEASURE_A timeout\n");
932 }
933 }
934
dramc_engine2_compare(u8 chn,enum dram_te_op wr)935 static void dramc_engine2_compare(u8 chn, enum dram_te_op wr)
936 {
937 u8 rank_status = ((read32(&ch[chn].ao.test2_3) & 0xf) == 1) ? 3 : 1;
938
939 if (wr == TE_OP_WRITE_READ_CHECK) {
940 dramc_engine2_check_complete(chn, rank_status);
941
942 SET32_BITFIELDS(&ch[chn].ao.test2_3, TEST2_3_TEST2W, 0,
943 TEST2_3_TEST2R, 0, TEST2_3_TEST1, 0);
944 udelay(1);
945 SET32_BITFIELDS(&ch[chn].ao.test2_3, TEST2_3_TEST2W, 1);
946 }
947
948 dramc_engine2_check_complete(chn, rank_status);
949 }
950
dramc_engine2_run(u8 chn,enum dram_te_op wr)951 static u32 dramc_engine2_run(u8 chn, enum dram_te_op wr)
952 {
953 u32 result;
954
955 if (wr == TE_OP_READ_CHECK) {
956 SET32_BITFIELDS(&ch[chn].ao.test2_4, TEST2_4_TESTAUDMODE, 0);
957 SET32_BITFIELDS(&ch[chn].ao.test2_3,
958 TEST2_3_TEST2W, 0, TEST2_3_TEST2R, 1, TEST2_3_TEST1, 0);
959 } else if (wr == TE_OP_WRITE_READ_CHECK) {
960 SET32_BITFIELDS(&ch[chn].ao.test2_3,
961 TEST2_3_TEST2W, 1, TEST2_3_TEST2R, 0, TEST2_3_TEST1, 0);
962 }
963
964 dramc_engine2_compare(chn, wr);
965
966 udelay(1);
967 result = read32(&ch[chn].nao.cmp_err);
968 SET32_BITFIELDS(&ch[chn].ao.test2_3,
969 TEST2_3_TEST2W, 0, TEST2_3_TEST2R, 0, TEST2_3_TEST1, 0);
970
971 return result;
972 }
973
dramc_engine2_end(u8 chn,u32 dummy_rd)974 static void dramc_engine2_end(u8 chn, u32 dummy_rd)
975 {
976 clrbits32(&ch[chn].ao.test2_4, 0x1 << 17);
977 write32(&ch[chn].ao.dummy_rd, dummy_rd);
978 }
979
dramc_find_gating_window(u32 result_r,u32 result_f,u32 * debug_cnt,u8 dly_coarse_large,u8 dly_coarse_0p5t,u8 * pass_begin,u8 * pass_count,u8 * pass_count_1,u8 * dly_fine_xt,u8 * dqs_high,u8 * dqs_done)980 static bool dramc_find_gating_window(u32 result_r, u32 result_f, u32 *debug_cnt,
981 u8 dly_coarse_large, u8 dly_coarse_0p5t, u8 *pass_begin, u8 *pass_count,
982 u8 *pass_count_1, u8 *dly_fine_xt, u8 *dqs_high, u8 *dqs_done)
983 {
984 bool find_tune = false;
985 u16 debug_cnt_perbyte, current_pass = 0, pass_byte_cnt = 0;
986
987 for (u8 dqs = 0; dqs < DQS_NUMBER; dqs++) {
988 u8 dqs_result_r = (u8)((result_r >> (8 * dqs)) & 0xff);
989 u8 dqs_result_f = (u8)((result_f >> (8 * dqs)) & 0xff);
990
991 if (pass_byte_cnt & (1 << dqs))
992 continue;
993 current_pass = 0;
994
995 debug_cnt_perbyte = (u16)debug_cnt[dqs];
996 if (dqs_result_r == 0 && dqs_result_f == 0 &&
997 debug_cnt_perbyte == GATING_GOLDEND_DQSCNT)
998 current_pass = 1;
999
1000 if (current_pass) {
1001 if (pass_begin[dqs] == 0) {
1002 pass_begin[dqs] = 1;
1003 pass_count_1[dqs] = 0;
1004 dramc_dbg("[Byte %d]First pass (%d, %d, %d)\n",
1005 dqs, dly_coarse_large, dly_coarse_0p5t, *dly_fine_xt);
1006 }
1007
1008 if (pass_begin[dqs] == 1)
1009 pass_count_1[dqs]++;
1010
1011 if (pass_begin[dqs] == 1 &&
1012 pass_count_1[dqs] * DQS_GW_FINE_STEP > DQS_GW_FINE_END) {
1013 dqs_high[dqs] = 0;
1014 dqs_done[dqs] = 1;
1015 }
1016
1017 if (pass_count_1[0] * DQS_GW_FINE_STEP > DQS_GW_FINE_END &&
1018 pass_count_1[1] * DQS_GW_FINE_STEP > DQS_GW_FINE_END) {
1019 dramc_dbg("All bytes gating window > 1 coarse_tune, Early break\n");
1020 *dly_fine_xt = DQS_GW_FINE_END;
1021 find_tune = true;
1022 }
1023 } else {
1024 if (pass_begin[dqs] != 1)
1025 continue;
1026
1027 dramc_dbg("[Byte %d] pass_begin[dqs]:%d, pass_count[dqs]:%d,pass_count_1:%d\n",
1028 dqs, pass_begin[dqs], pass_count[dqs], pass_count_1[dqs]);
1029
1030 pass_begin[dqs] = 0;
1031 if (pass_count_1[dqs] > pass_count[dqs]) {
1032 pass_count[dqs] = pass_count_1[dqs];
1033 if (pass_count_1[dqs] * DQS_GW_FINE_STEP > 32 &&
1034 pass_count_1[dqs] * DQS_GW_FINE_STEP < 96)
1035 pass_byte_cnt |= (1 << dqs);
1036 if (pass_byte_cnt == 3) {
1037 *dly_fine_xt = DQS_GW_FINE_END;
1038 find_tune = true;
1039 }
1040 }
1041 }
1042 }
1043
1044 return find_tune;
1045 }
1046
dramc_find_dly_tune(u8 chn,u8 dly_coarse_large,u8 dly_coarse_0p5t,u8 dly_fine_xt,u8 * dqs_high,u8 * dly_coarse_large_cnt,u8 * dly_coarse_0p5t_cnt,u8 * dly_fine_tune_cnt,u8 * dqs_trans,u8 * dqs_done)1047 static void dramc_find_dly_tune(u8 chn, u8 dly_coarse_large, u8 dly_coarse_0p5t,
1048 u8 dly_fine_xt, u8 *dqs_high, u8 *dly_coarse_large_cnt,
1049 u8 *dly_coarse_0p5t_cnt, u8 *dly_fine_tune_cnt, u8 *dqs_trans, u8 *dqs_done)
1050 {
1051 for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) {
1052 u32 dqs_cnt = read32(&ch[chn].phy_nao.misc_phy_stben_b[dqs]);
1053 dqs_cnt = (dqs_cnt >> 16) & 3;
1054
1055 if (dqs_done[dqs] == 1)
1056 continue;
1057
1058 if (dqs_cnt == 3)
1059 dqs_high[dqs]++;
1060
1061 if (dqs_high[dqs] * DQS_GW_FINE_STEP <= 16)
1062 continue;
1063
1064 switch (dqs_cnt) {
1065 case 3:
1066 dly_coarse_large_cnt[dqs] = dly_coarse_large;
1067 dly_coarse_0p5t_cnt[dqs] = dly_coarse_0p5t;
1068 dly_fine_tune_cnt[dqs] = dly_fine_xt;
1069 dqs_trans[dqs] = 1;
1070 break;
1071 case 2:
1072 case 1:
1073 if (dqs_trans[dqs] == 1)
1074 dramc_dbg("[Byte %ld] Lead/lag falling Transition"
1075 " (%d, %d, %d)\n",
1076 dqs, dly_coarse_large_cnt[dqs],
1077 dly_coarse_0p5t_cnt[dqs], dly_fine_tune_cnt[dqs]);
1078 dqs_trans[dqs]++;
1079 break;
1080 case 0:
1081 dramc_dbg("[Byte %ld] Lead/lag Transition tap number (%d)\n",
1082 dqs, dqs_trans[dqs]);
1083 dqs_high[dqs] = 0;
1084 break;
1085 }
1086 }
1087 }
1088
dram_phy_reset(u8 chn)1089 static void dram_phy_reset(u8 chn)
1090 {
1091 SET32_BITFIELDS(&ch[chn].ao.ddrconf0, DDRCONF0_RDATRST, 1);
1092 SET32_BITFIELDS(&ch[chn].phy.misc_ctrl1, MISC_CTRL1_R_DMPHYRST, 1);
1093 clrbits32(&ch[chn].phy.b[0].dq[9], (1 << 4) | (1 << 0));
1094 clrbits32(&ch[chn].phy.b[1].dq[9], (1 << 4) | (1 << 0));
1095
1096 udelay(1);
1097 setbits32(&ch[chn].phy.b[1].dq[9], (1 << 4) | (1 << 0));
1098 setbits32(&ch[chn].phy.b[0].dq[9], (1 << 4) | (1 << 0));
1099 SET32_BITFIELDS(&ch[chn].phy.misc_ctrl1, MISC_CTRL1_R_DMPHYRST, 0);
1100 SET32_BITFIELDS(&ch[chn].ao.ddrconf0, DDRCONF0_RDATRST, 0);
1101 }
1102
dramc_set_gating_mode(u8 chn,bool mode)1103 static void dramc_set_gating_mode(u8 chn, bool mode)
1104 {
1105 u8 vref = 0, burst = 0;
1106
1107 if (mode) {
1108 vref = 2;
1109 burst = 1;
1110 }
1111
1112 for (size_t b = 0; b < 2; b++) {
1113 clrsetbits32(&ch[chn].phy.b[b].dq[6], 0x3 << 14, vref << 14);
1114 setbits32(&ch[chn].phy.b[b].dq[9], 0x1 << 5);
1115 }
1116
1117 clrsetbits32(&ch[chn].ao.stbcal1, 0x1 << 5, burst << 5);
1118 setbits32(&ch[chn].ao.stbcal, 0x1 << 30);
1119
1120 clrbits32(&ch[chn].phy.b[0].dq[9], (0x1 << 4) | (0x1 << 0));
1121 clrbits32(&ch[chn].phy.b[1].dq[9], (0x1 << 4) | (0x1 << 0));
1122 udelay(1);
1123 setbits32(&ch[chn].phy.b[1].dq[9], (0x1 << 4) | (0x1 << 0));
1124 setbits32(&ch[chn].phy.b[0].dq[9], (0x1 << 4) | (0x1 << 0));
1125 }
1126
dramc_rx_dqs_gating_cal_pre(u8 chn,u8 rank)1127 static void dramc_rx_dqs_gating_cal_pre(u8 chn, u8 rank)
1128 {
1129 SET32_BITFIELDS(&ch[chn].ao.refctrl0, REFCTRL0_PBREFEN, 0);
1130
1131 dramc_hw_gating_onoff(chn, false);
1132
1133 SET32_BITFIELDS(&ch[chn].ao.stbcal1, STBCAL1_STBENCMPEN, 1);
1134 SET32_BITFIELDS(&ch[chn].ao.stbcal1, STBCAL1_STBCNT_LATCH_EN, 1);
1135 SET32_BITFIELDS(&ch[chn].ao.ddrconf0, DDRCONF0_DM4TO1MODE, 0);
1136 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSGCNTEN, 1);
1137
1138 udelay(4);
1139 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSGCNTRST, 1);
1140 udelay(1);
1141 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSGCNTRST, 0);
1142 SET32_BITFIELDS(&ch[chn].phy.misc_ctrl1, MISC_CTRL1_R_DMSTBENCMP_RK,
1143 rank);
1144 }
1145
set_selph_gating_value(uint32_t * addr,u8 dly,u8 dly_p1)1146 static void set_selph_gating_value(uint32_t *addr, u8 dly, u8 dly_p1)
1147 {
1148 clrsetbits32(addr, 0x77777777,
1149 (dly << 0) | (dly << 8) | (dly << 16) | (dly << 24) |
1150 (dly_p1 << 4) | (dly_p1 << 12) | (dly_p1 << 20) | (dly_p1 << 28));
1151 }
1152
dramc_write_dqs_gating_result(u8 chn,u8 rank,u8 * best_coarse_tune2t,u8 * best_coarse_tune0p5t,u8 * best_coarse_tune2t_p1,u8 * best_coarse_tune0p5t_p1)1153 static void dramc_write_dqs_gating_result(u8 chn, u8 rank,
1154 u8 *best_coarse_tune2t, u8 *best_coarse_tune0p5t,
1155 u8 *best_coarse_tune2t_p1, u8 *best_coarse_tune0p5t_p1)
1156 {
1157 u8 best_coarse_rodt[DQS_NUMBER], best_coarse_0p5t_rodt[DQS_NUMBER];
1158 u8 best_coarse_rodt_p1[DQS_NUMBER];
1159 u8 best_coarse_0p5t_rodt_p1[DQS_NUMBER];
1160
1161 dramc_rx_dqs_isi_pulse_cg_switch(chn, true);
1162
1163 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0,
1164 0x77777777,
1165 (best_coarse_tune2t[0] << 0) | (best_coarse_tune2t[1] << 8) |
1166 (best_coarse_tune2t_p1[0] << 4) | (best_coarse_tune2t_p1[1] << 12));
1167 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg1,
1168 0x77777777,
1169 (best_coarse_tune0p5t[0] << 0) | (best_coarse_tune0p5t[1] << 8) |
1170 (best_coarse_tune0p5t_p1[0] << 4) | (best_coarse_tune0p5t_p1[1] << 12));
1171
1172 for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) {
1173 u8 tmp_value = (best_coarse_tune2t[dqs] << 3)
1174 + best_coarse_tune0p5t[dqs];
1175
1176 if (tmp_value >= 11) {
1177 tmp_value -= 11;
1178 best_coarse_rodt[dqs] = tmp_value >> 3;
1179 best_coarse_0p5t_rodt[dqs] =
1180 tmp_value - (best_coarse_rodt[dqs] << 3);
1181
1182 tmp_value = (best_coarse_tune2t_p1[dqs] << 3) +
1183 best_coarse_tune0p5t_p1[dqs] - 11;
1184 best_coarse_rodt_p1[dqs] = tmp_value >> 3;
1185 best_coarse_0p5t_rodt_p1[dqs] =
1186 tmp_value - (best_coarse_rodt_p1[dqs] << 3);
1187
1188 dramc_dbg("Best RODT dly(2T, 0.5T) = (%d, %d)\n",
1189 best_coarse_rodt[dqs],
1190 best_coarse_0p5t_rodt[dqs]);
1191 } else {
1192 best_coarse_rodt[dqs] = 0;
1193 best_coarse_0p5t_rodt[dqs] = 0;
1194 best_coarse_rodt_p1[dqs] = 4;
1195 best_coarse_0p5t_rodt_p1[dqs] = 4;
1196 dramc_dbg("RxdqsGatingCal error: best_coarse_tune2t:%zd"
1197 " is already 0. RODT cannot be -1 coarse\n",
1198 dqs);
1199 }
1200 }
1201
1202 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_odten0,
1203 0x77777777,
1204 (best_coarse_rodt[0] << 0) | (best_coarse_rodt[1] << 8) |
1205 (best_coarse_rodt_p1[0] << 4) | (best_coarse_rodt_p1[1] << 12));
1206 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_odten1,
1207 0x77777777,
1208 (best_coarse_0p5t_rodt[0] << 0) | (best_coarse_0p5t_rodt[1] << 8) |
1209 (best_coarse_0p5t_rodt_p1[0] << 4) | (best_coarse_0p5t_rodt_p1[1] << 12));
1210 }
1211
dramc_rx_dqs_gating_cal_partial(u8 chn,u8 rank,u32 coarse_start,u32 coarse_end,u8 freqDiv,u8 * pass_begin,u8 * pass_count,u8 * pass_count_1,u8 * dqs_done,u8 * dqs_high,u8 * dqs_transition,u8 * dly_coarse_large_cnt,u8 * dly_coarse_0p5t_cnt,u8 * dly_fine_tune_cnt)1212 static void dramc_rx_dqs_gating_cal_partial(u8 chn, u8 rank,
1213 u32 coarse_start, u32 coarse_end, u8 freqDiv,
1214 u8 *pass_begin, u8 *pass_count, u8 *pass_count_1, u8 *dqs_done,
1215 u8 *dqs_high, u8 *dqs_transition, u8 *dly_coarse_large_cnt,
1216 u8 *dly_coarse_0p5t_cnt, u8 *dly_fine_tune_cnt)
1217 {
1218 u8 dqs;
1219 u32 debug_cnt[DQS_NUMBER];
1220
1221 for (u32 coarse_tune = coarse_start; coarse_tune < coarse_end;
1222 coarse_tune++) {
1223 u32 dly_coarse_large_rodt = 0, dly_coarse_0p5t_rodt = 0;
1224 u32 dly_coarse_large_rodt_p1 = 4, dly_coarse_0p5t_rodt_p1 = 4;
1225
1226 u8 dly_coarse_large = coarse_tune / RX_DQS_CTL_LOOP;
1227 u8 dly_coarse_0p5t = coarse_tune % RX_DQS_CTL_LOOP;
1228 u32 dly_coarse_large_p1 = (coarse_tune + freqDiv) / RX_DQS_CTL_LOOP;
1229 u32 dly_coarse_0p5t_p1 = (coarse_tune + freqDiv) % RX_DQS_CTL_LOOP;
1230 u32 value = (dly_coarse_large << 3) + dly_coarse_0p5t;
1231
1232 if (value >= 11) {
1233 value -= 11;
1234 dly_coarse_large_rodt = value >> 3;
1235 dly_coarse_0p5t_rodt =
1236 value - (dly_coarse_large_rodt << 3);
1237 value = (dly_coarse_large << 3) + dly_coarse_0p5t - 11;
1238 dly_coarse_large_rodt_p1 = value >> 3;
1239 dly_coarse_0p5t_rodt_p1 =
1240 value - (dly_coarse_large_rodt_p1 << 3);
1241 }
1242
1243 set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0,
1244 dly_coarse_large, dly_coarse_large_p1);
1245 set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_dqsg1,
1246 dly_coarse_0p5t, dly_coarse_0p5t_p1);
1247 set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_odten0,
1248 dly_coarse_large_rodt, dly_coarse_large_rodt_p1);
1249 set_selph_gating_value(&ch[chn].ao.shu[0].rk[rank].selph_odten1,
1250 dly_coarse_0p5t_rodt, dly_coarse_0p5t_rodt_p1);
1251
1252 for (u8 dly_fine_xt = 0; dly_fine_xt < DQS_GW_FINE_END;
1253 dly_fine_xt += 4) {
1254 dramc_set_gating_mode(chn, 0);
1255 WRITE32_BITFIELDS(&ch[chn].ao.shu[0].rk[rank].dqsien,
1256 SHURK_DQSIEN_DQS0IEN, dly_fine_xt,
1257 SHURK_DQSIEN_DQS1IEN, dly_fine_xt);
1258
1259 dram_phy_reset(chn);
1260 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSGCNTRST, 1);
1261 udelay(1);
1262 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSGCNTRST, 0);
1263
1264 dramc_engine2_run(chn, TE_OP_READ_CHECK);
1265
1266 u32 result_r = READ32_BITFIELD(
1267 &ch[chn].phy.misc_stberr_rk0_r,
1268 MISC_STBERR_RK_R_STBERR_RK_R);
1269 u32 result_f = READ32_BITFIELD(
1270 &ch[chn].phy.misc_stberr_rk0_f,
1271 MISC_STBERR_RK_F_STBERR_RK_F);
1272
1273 debug_cnt[0] = read32(&ch[chn].nao.dqsgnwcnt[0]);
1274 debug_cnt[1] = (debug_cnt[0] >> 16) & 0xffff;
1275 debug_cnt[0] &= 0xffff;
1276
1277 dramc_set_gating_mode(chn, 1);
1278 dramc_engine2_run(chn, TE_OP_READ_CHECK);
1279
1280 dramc_find_dly_tune(chn, dly_coarse_large,
1281 dly_coarse_0p5t, dly_fine_xt, dqs_high,
1282 dly_coarse_large_cnt, dly_coarse_0p5t_cnt,
1283 dly_fine_tune_cnt, dqs_transition, dqs_done);
1284
1285 dramc_dbg("%d %d %d |", dly_coarse_large,
1286 dly_coarse_0p5t, dly_fine_xt);
1287 for (dqs = 0; dqs < DQS_NUMBER; dqs++)
1288 dramc_dbg("%X ", debug_cnt[dqs]);
1289
1290 dramc_dbg(" |");
1291 for (dqs = 0; dqs < DQS_NUMBER; dqs++) {
1292 dramc_dbg("(%X %X)",
1293 (result_f >> (DQS_BIT_NUMBER * dqs)) & 0xff,
1294 (result_r >> (DQS_BIT_NUMBER * dqs)) & 0xff);
1295 }
1296
1297 dramc_dbg("\n");
1298 if (dramc_find_gating_window(result_r, result_f, debug_cnt,
1299 dly_coarse_large, dly_coarse_0p5t, pass_begin,
1300 pass_count, pass_count_1, &dly_fine_xt,
1301 dqs_high, dqs_done))
1302 coarse_tune = coarse_end;
1303 }
1304 }
1305 }
1306
dramc_rx_dqs_gating_cal(u8 chn,u8 rank,u8 freq_group,const struct sdram_params * params,const bool fast_calib,const struct mr_value * mr)1307 static void dramc_rx_dqs_gating_cal(u8 chn, u8 rank, u8 freq_group,
1308 const struct sdram_params *params, const bool fast_calib,
1309 const struct mr_value *mr)
1310 {
1311 u8 dqs, fsp, freqDiv = 4;
1312 u8 pass_begin[DQS_NUMBER] = {0}, pass_count[DQS_NUMBER] = {0},
1313 pass_count_1[DQS_NUMBER] = {0}, dqs_done[DQS_NUMBER] = {0};
1314 u8 min_coarse_tune2t[DQS_NUMBER], min_coarse_tune0p5t[DQS_NUMBER],
1315 min_fine_tune[DQS_NUMBER];
1316 u8 best_fine_tune[DQS_NUMBER], best_coarse_tune0p5t[DQS_NUMBER],
1317 best_coarse_tune2t[DQS_NUMBER];
1318 u8 best_coarse_tune0p5t_p1[DQS_NUMBER], best_coarse_tune2t_p1[DQS_NUMBER];
1319 u8 dqs_high[DQS_NUMBER] = {0}, dqs_transition[DQS_NUMBER] = {0};
1320 u8 dly_coarse_large_cnt[DQS_NUMBER] = {0}, dly_coarse_0p5t_cnt[DQS_NUMBER] = {0},
1321 dly_fine_tune_cnt[DQS_NUMBER] = {0};
1322 u32 coarse_start, coarse_end;
1323
1324 struct reg_value regs_bak[] = {
1325 {&ch[chn].ao.stbcal},
1326 {&ch[chn].ao.stbcal1},
1327 {&ch[chn].ao.ddrconf0},
1328 {&ch[chn].ao.spcmd},
1329 {&ch[chn].ao.refctrl0},
1330 {&ch[chn].phy.b[0].dq[6]},
1331 {&ch[chn].phy.b[1].dq[6]},
1332 };
1333 for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++)
1334 regs_bak[i].value = read32(regs_bak[i].addr);
1335
1336 fsp = get_freq_fsq(freq_group);
1337 dramc_rx_dqs_isi_pulse_cg_switch(chn, false);
1338
1339 dramc_mode_reg_write_by_rank(chn, rank, 0x1, mr->MR01Value[fsp] | 0x80);
1340 dramc_rx_dqs_gating_cal_pre(chn, rank);
1341
1342 u32 dummy_rd_backup = read32(&ch[chn].ao.dummy_rd);
1343 dramc_engine2_init(chn, rank, TEST2_1_CAL, 0xaa000023, true);
1344
1345 switch (freq_group) {
1346 case LP4X_DDR1600:
1347 coarse_start = 18;
1348 break;
1349 case LP4X_DDR2400:
1350 coarse_start = 25;
1351 break;
1352 case LP4X_DDR3200:
1353 coarse_start = 25;
1354 break;
1355 case LP4X_DDR3600:
1356 coarse_start = 21;
1357 break;
1358 default:
1359 die("Invalid DDR frequency group %u\n", freq_group);
1360 return;
1361 }
1362 coarse_end = coarse_start + 12;
1363
1364 dramc_dbg("[Gating]\n");
1365
1366 if (!fast_calib) {
1367 dramc_rx_dqs_gating_cal_partial(chn, rank,
1368 coarse_start, coarse_end,
1369 freqDiv, pass_begin, pass_count, pass_count_1, dqs_done,
1370 dqs_high, dqs_transition, dly_coarse_large_cnt,
1371 dly_coarse_0p5t_cnt, dly_fine_tune_cnt);
1372 dramc_engine2_end(chn, dummy_rd_backup);
1373 }
1374
1375 for (dqs = 0; dqs < DQS_NUMBER; dqs++) {
1376 if (fast_calib) {
1377 dramc_dbg("[bypass Gating params] dqs: %d\n", dqs);
1378 pass_count[dqs] = params->gating_pass_count[chn][rank][dqs];
1379 min_fine_tune[dqs] = params->gating_fine_tune[chn][rank][dqs];
1380 min_coarse_tune0p5t[dqs] = params->gating05T[chn][rank][dqs];
1381 min_coarse_tune2t[dqs] = params->gating2T[chn][rank][dqs];
1382 } else {
1383 pass_count[dqs] = dqs_transition[dqs];
1384 min_fine_tune[dqs] = dly_fine_tune_cnt[dqs];
1385 min_coarse_tune0p5t[dqs] = dly_coarse_0p5t_cnt[dqs];
1386 min_coarse_tune2t[dqs] = dly_coarse_large_cnt[dqs];
1387 }
1388 u8 tmp_offset = pass_count[dqs] * DQS_GW_FINE_STEP / 2;
1389 u8 tmp_value = min_fine_tune[dqs] + tmp_offset;
1390 best_fine_tune[dqs] = tmp_value % RX_DLY_DQSIENSTB_LOOP;
1391
1392 tmp_offset = tmp_value / RX_DLY_DQSIENSTB_LOOP;
1393 tmp_value = min_coarse_tune0p5t[dqs] + tmp_offset;
1394 best_coarse_tune0p5t[dqs] = tmp_value % RX_DQS_CTL_LOOP;
1395
1396 tmp_offset = tmp_value / RX_DQS_CTL_LOOP;
1397 best_coarse_tune2t[dqs] = min_coarse_tune2t[dqs] + tmp_offset;
1398
1399 tmp_value = best_coarse_tune0p5t[dqs] + freqDiv;
1400 best_coarse_tune0p5t_p1[dqs] = tmp_value % RX_DQS_CTL_LOOP;
1401
1402 tmp_offset = tmp_value / RX_DQS_CTL_LOOP;
1403 best_coarse_tune2t_p1[dqs] =
1404 best_coarse_tune2t[dqs] + tmp_offset;
1405 }
1406
1407 for (dqs = 0; dqs < DQS_NUMBER; dqs++)
1408 dramc_dbg("Best DQS%d dly(2T, 0.5T, fine tune)"
1409 " = (%d, %d, %d)\n", dqs, best_coarse_tune2t[dqs],
1410 best_coarse_tune0p5t[dqs], best_fine_tune[dqs]);
1411
1412 for (dqs = 0; dqs < DQS_NUMBER; dqs++)
1413 dramc_dbg("Best DQS%d P1 dly(2T, 0.5T, fine tune)"
1414 " = (%d, %d, %d)\n", dqs, best_coarse_tune2t_p1[dqs],
1415 best_coarse_tune0p5t_p1[dqs], best_fine_tune[dqs]);
1416
1417 for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++)
1418 write32(regs_bak[i].addr, regs_bak[i].value);
1419
1420 dramc_mode_reg_write_by_rank(chn, rank, 0x1, mr->MR01Value[fsp]);
1421
1422 dramc_write_dqs_gating_result(chn, rank, best_coarse_tune2t,
1423 best_coarse_tune0p5t, best_coarse_tune2t_p1, best_coarse_tune0p5t_p1);
1424
1425 WRITE32_BITFIELDS(&ch[chn].ao.shu[0].rk[rank].dqsien,
1426 SHURK_DQSIEN_DQS0IEN, best_fine_tune[0],
1427 SHURK_DQSIEN_DQS1IEN, best_fine_tune[1]);
1428
1429 dram_phy_reset(chn);
1430 }
1431
dramc_rx_rd_dqc_init(u8 chn,u8 rank)1432 static void dramc_rx_rd_dqc_init(u8 chn, u8 rank)
1433 {
1434 const u8 *lpddr_phy_mapping = phy_mapping[chn];
1435 u16 temp_value = 0;
1436
1437 for (size_t b = 0; b < 2; b++)
1438 clrbits32(&ch[chn].phy.shu[0].b[b].dq[7], 0x1 << 7);
1439
1440 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, rank);
1441 SET32_BITFIELDS(&ch[chn].ao.mpc_option, MPC_OPTION_MPCRKEN, 1);
1442
1443 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++)
1444 temp_value |= ((0x5555 >> bit) & 0x1) << lpddr_phy_mapping[bit];
1445
1446 u16 mr15_golden_value = temp_value & 0xff;
1447 u16 mr20_golden_value = (temp_value >> 8) & 0xff;
1448 SET32_BITFIELDS(&ch[chn].ao.mr_golden,
1449 MR_GOLDEN_MR15_GOLDEN, mr15_golden_value,
1450 MR_GOLDEN_MR20_GOLDEN, mr20_golden_value);
1451 }
1452
dramc_rx_rd_dqc_run(u8 chn)1453 static u32 dramc_rx_rd_dqc_run(u8 chn)
1454 {
1455 u32 loop = 0;
1456 SET32_BITFIELDS(&ch[chn].ao.spcmdctrl, SPCMDCTRL_RDDQCDIS, 1);
1457 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_RDDQCEN, 1);
1458
1459 while (!wait_us(10, read32(&ch[chn].nao.spcmdresp) & (0x1 << 7))) {
1460 if (loop++ > 10)
1461 dramc_dbg("[RDDQC] resp fail (time out)\n");
1462 }
1463
1464 u32 result = read32(&ch[chn].nao.rdqc_cmp);
1465 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_RDDQCEN, 0);
1466 SET32_BITFIELDS(&ch[chn].ao.spcmdctrl, SPCMDCTRL_RDDQCDIS, 0);
1467
1468 return result;
1469 }
1470
dramc_rx_rd_dqc_end(u8 chn)1471 static void dramc_rx_rd_dqc_end(u8 chn)
1472 {
1473 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, 0);
1474 }
1475
dramc_rx_vref_pre_setting(u8 chn)1476 static void dramc_rx_vref_pre_setting(u8 chn)
1477 {
1478 SET32_BITFIELDS(&ch[chn].phy.b[0].dq[5], B0_DQ5_RG_RX_ARDQ_VREF_EN_B0, 1);
1479 SET32_BITFIELDS(&ch[chn].phy.b[1].dq[5], B1_DQ5_RG_RX_ARDQ_VREF_EN_B1, 1);
1480 }
1481
dramc_set_rx_vref(u8 chn,u8 vref)1482 static void dramc_set_rx_vref(u8 chn, u8 vref)
1483 {
1484 for (size_t b = 0; b < 2; b++)
1485 SET32_BITFIELDS(&ch[chn].phy.shu[0].b[b].dq[5],
1486 SHU1_BX_DQ5_RG_RX_ARDQ_VREF_SEL_B0, vref);
1487 dramc_dbg("set rx vref :%d\n", vref);
1488 }
1489
dramc_set_tx_vref(u8 chn,u8 rank,u8 value)1490 static void dramc_set_tx_vref(u8 chn, u8 rank, u8 value)
1491 {
1492 dramc_mode_reg_write_by_rank(chn, rank, 14, value);
1493 }
1494
dramc_set_vref(u8 chn,u8 rank,enum CAL_TYPE type,u8 vref)1495 static void dramc_set_vref(u8 chn, u8 rank, enum CAL_TYPE type, u8 vref)
1496 {
1497 if (type == RX_WIN_TEST_ENG)
1498 dramc_set_rx_vref(chn, vref);
1499 else
1500 dramc_set_tx_vref(chn, rank, vref);
1501 }
1502
dramc_transfer_dly_tune(u8 chn,u32 dly,u32 adjust_center,struct tx_dly_tune * dly_tune)1503 static void dramc_transfer_dly_tune(u8 chn, u32 dly, u32 adjust_center,
1504 struct tx_dly_tune *dly_tune)
1505 {
1506 u8 tune = 3, fine_tune = 0;
1507 u16 tmp;
1508
1509 fine_tune = dly & (TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP - 1);
1510 tmp = (dly / TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP) << 1;
1511
1512 if (adjust_center) {
1513 if (fine_tune < 10) {
1514 fine_tune += TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP >> 1;
1515 tmp--;
1516 } else if (fine_tune > TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP - 10) {
1517 fine_tune -= TX_DQ_COARSE_TUNE_TO_FINE_TUNE_TAP >> 1;
1518 tmp++;
1519 }
1520 }
1521
1522 dly_tune->fine_tune = fine_tune;
1523 dly_tune->coarse_tune_small = tmp - ((tmp >> tune) << tune);
1524 dly_tune->coarse_tune_large = tmp >> tune;
1525
1526 tmp -= 3;
1527 dly_tune->coarse_tune_small_oen = tmp - ((tmp >> tune) << tune);
1528 dly_tune->coarse_tune_large_oen = tmp >> tune;
1529 }
1530
dramc_set_rx_dly_factor(u8 chn,u8 rank,enum RX_TYPE type,u32 val)1531 static void dramc_set_rx_dly_factor(u8 chn, u8 rank, enum RX_TYPE type, u32 val)
1532 {
1533 int b, dq;
1534
1535 switch (type) {
1536 case RX_DQ:
1537 for (dq = 2; dq < 6; dq++)
1538 for (b = 0; b < 2; b++)
1539 WRITE32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[b].dq[dq],
1540 SHU1_R0_B0_DQ6_RK0_RX_ARDQM0_F_DLY_B0, val,
1541 SHU1_R0_B0_DQ6_RK0_RX_ARDQM0_R_DLY_B0, val,
1542 SHU1_R0_B0_DQ6_RK0_RX_ARDQS0_F_DLY_B0, val,
1543 SHU1_R0_B0_DQ6_RK0_RX_ARDQS0_R_DLY_B0, val);
1544 break;
1545
1546 case RX_DQM:
1547 for (b = 0; b < 2; b++)
1548 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[b].dq[6],
1549 SHU1_R0_B0_DQ6_RK0_RX_ARDQM0_F_DLY_B0, val,
1550 SHU1_R0_B0_DQ6_RK0_RX_ARDQM0_R_DLY_B0, val);
1551 break;
1552
1553 case RX_DQS:
1554 for (b = 0; b < 2; b++)
1555 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[b].dq[6],
1556 SHU1_R0_B0_DQ6_RK0_RX_ARDQS0_F_DLY_B0, val,
1557 SHU1_R0_B0_DQ6_RK0_RX_ARDQS0_R_DLY_B0, val);
1558 break;
1559 default:
1560 dramc_err("error calibration type: %d\n", type);
1561 break;
1562 }
1563 }
1564
dramc_set_tx_dly_factor(u8 chn,u8 rk,enum CAL_TYPE type,u8 * dq_small_reg,u32 dly)1565 static void dramc_set_tx_dly_factor(u8 chn, u8 rk,
1566 enum CAL_TYPE type, u8 *dq_small_reg, u32 dly)
1567 {
1568 struct tx_dly_tune dly_tune = {0};
1569 u32 dly_large = 0, dly_large_oen = 0, dly_small = 0, dly_small_oen = 0;
1570 u32 adjust_center = 0;
1571
1572 dramc_transfer_dly_tune(chn, dly, adjust_center, &dly_tune);
1573
1574 for (u8 i = 0; i < 4; i++) {
1575 dly_large += dly_tune.coarse_tune_large << (i * 4);
1576 dly_large_oen += dly_tune.coarse_tune_large_oen << (i * 4);
1577 dly_small += dly_tune.coarse_tune_small << (i * 4);
1578 dly_small_oen += dly_tune.coarse_tune_small_oen << (i * 4);
1579 }
1580
1581 if (type == TX_WIN_DQ_DQM)
1582 dramc_dbg("%3d |%d %d %2d | [0]",
1583 dly, dly_tune.coarse_tune_large,
1584 dly_tune.coarse_tune_small, dly_tune.fine_tune);
1585
1586 if (*dq_small_reg != dly_tune.coarse_tune_small) {
1587 if (type == TX_WIN_DQ_DQM || type == TX_WIN_DQ_ONLY) {
1588 clrsetbits32(&ch[chn].ao.shu[0].rk[rk].selph_dq[0],
1589 0x77777777, dly_large | (dly_large_oen << 16));
1590 clrsetbits32(&ch[chn].ao.shu[0].rk[rk].selph_dq[2],
1591 0x77777777, dly_small | (dly_small_oen << 16));
1592 }
1593
1594 if (type == TX_WIN_DQ_DQM) {
1595 /* Large coarse_tune setting */
1596 clrsetbits32(&ch[chn].ao.shu[0].rk[rk].selph_dq[1],
1597 0x77777777, dly_large | (dly_large_oen << 16));
1598 clrsetbits32(&ch[chn].ao.shu[0].rk[rk].selph_dq[3],
1599 0x77777777, dly_small | (dly_small_oen << 16));
1600 }
1601 }
1602 *dq_small_reg = dly_tune.coarse_tune_small;
1603
1604 if (type == TX_WIN_DQ_DQM || type == TX_WIN_DQ_ONLY) {
1605 for (size_t b = 0; b < 2; b++)
1606 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rk].b[b].dq[7],
1607 FINE_TUNE_DQ, dly_tune.fine_tune);
1608 }
1609 if (type == TX_WIN_DQ_DQM) {
1610 for (size_t b = 0; b < 2; b++)
1611 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rk].b[b].dq[7],
1612 FINE_TUNE_DQM, dly_tune.fine_tune);
1613 }
1614 }
1615
dramc_get_smallest_dqs_dly(u8 chn,u8 rank,const struct sdram_params * params)1616 static u32 dramc_get_smallest_dqs_dly(u8 chn, u8 rank, const struct sdram_params *params)
1617 {
1618 const u8 mck = 3;
1619 u32 min_dly = 0xffff, virtual_delay = 0;
1620 u32 tx_dly = read32(&ch[chn].ao.shu[0].selph_dqs0);
1621 u32 dly = read32(&ch[chn].ao.shu[0].selph_dqs1);
1622 u32 tmp;
1623
1624 for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) {
1625 tmp = ((tx_dly >> (dqs << 2) & 0x7) << mck) +
1626 (dly >> (dqs << 2) & 0x7);
1627 virtual_delay = (tmp << 5) + params->wr_level[chn][rank][dqs];
1628 min_dly = MIN(min_dly, virtual_delay);
1629 }
1630
1631 return min_dly;
1632 }
1633
dramc_get_dly_range(u8 chn,u8 rank,enum CAL_TYPE type,u8 freq_group,u16 * pre_cal,s16 * begin,s16 * end,const struct sdram_params * params)1634 static void dramc_get_dly_range(u8 chn, u8 rank, enum CAL_TYPE type,
1635 u8 freq_group, u16 *pre_cal, s16 *begin, s16 *end,
1636 const struct sdram_params *params)
1637 {
1638 u16 pre_dq_dly;
1639 switch (type) {
1640 case RX_WIN_RD_DQC:
1641 case RX_WIN_TEST_ENG:
1642 switch (freq_group) {
1643 case LP4X_DDR1600:
1644 *begin = -48;
1645 break;
1646 case LP4X_DDR2400:
1647 *begin = -30;
1648 break;
1649 case LP4X_DDR3200:
1650 case LP4X_DDR3600:
1651 *begin = -26;
1652 break;
1653 default:
1654 die("Invalid DDR frequency group %u\n", freq_group);
1655 return;
1656 }
1657
1658 *end = MAX_RX_DQDLY_TAPS;
1659 break;
1660
1661 case TX_WIN_DQ_DQM:
1662 *begin = dramc_get_smallest_dqs_dly(chn, rank, params);
1663 *end = *begin + 256;
1664 break;
1665
1666 case TX_WIN_DQ_ONLY:
1667 pre_dq_dly = MIN(pre_cal[0], pre_cal[1]);
1668 pre_dq_dly = (pre_dq_dly > 24) ? (pre_dq_dly - 24) : 0;
1669 *begin = pre_dq_dly;
1670 *end = *begin + 64;
1671 break;
1672 default:
1673 dramc_err("error calibration type: %d\n", type);
1674 break;
1675 }
1676 }
1677
dramc_check_dqdqs_win(struct win_perbit_dly * perbit_dly,s16 dly,s16 dly_end,bool fail_bit)1678 static int dramc_check_dqdqs_win(struct win_perbit_dly *perbit_dly,
1679 s16 dly, s16 dly_end, bool fail_bit)
1680 {
1681 int pass_win = 0;
1682
1683 if (perbit_dly->first_pass == PASS_RANGE_NA) {
1684 if (!fail_bit) /* compare correct: pass */
1685 perbit_dly->first_pass = dly;
1686 } else if (perbit_dly->last_pass == PASS_RANGE_NA) {
1687 if (fail_bit) /* compare error: fail */
1688 perbit_dly->last_pass = dly - 1;
1689 else if (dly == dly_end)
1690 perbit_dly->last_pass = dly;
1691
1692 if (perbit_dly->last_pass != PASS_RANGE_NA) {
1693 pass_win = perbit_dly->last_pass - perbit_dly->first_pass;
1694 int best_pass_win = perbit_dly->best_last - perbit_dly->best_first;
1695 if (pass_win >= best_pass_win) {
1696 perbit_dly->best_last = perbit_dly->last_pass;
1697 perbit_dly->best_first = perbit_dly->first_pass;
1698 }
1699
1700 /* Clear to find the next pass range if it has */
1701 perbit_dly->first_pass = PASS_RANGE_NA;
1702 perbit_dly->last_pass = PASS_RANGE_NA;
1703 }
1704 }
1705
1706 return pass_win;
1707 }
1708
dramc_set_vref_dly(struct vref_perbit_dly * vref_dly,struct win_perbit_dly delay[])1709 static void dramc_set_vref_dly(struct vref_perbit_dly *vref_dly, struct win_perbit_dly delay[])
1710 {
1711 struct win_perbit_dly *perbit_dly = vref_dly->perbit_dly;
1712
1713 for (u8 bit = 0; bit < DQ_DATA_WIDTH; bit++) {
1714 delay[bit].win_center = (delay[bit].best_first + delay[bit].best_last) >> 1;
1715
1716 perbit_dly[bit].best_first = delay[bit].best_first;
1717 perbit_dly[bit].best_last = delay[bit].best_last;
1718 perbit_dly[bit].win_center = delay[bit].win_center;
1719 perbit_dly[bit].best_dqdly = delay[bit].best_dqdly;
1720 }
1721 }
1722
dramk_calc_best_vref(enum CAL_TYPE type,u8 vref,struct vref_perbit_dly * vref_dly,struct win_perbit_dly delay[],u32 * win_min_max)1723 static bool dramk_calc_best_vref(enum CAL_TYPE type, u8 vref,
1724 struct vref_perbit_dly *vref_dly, struct win_perbit_dly delay[],
1725 u32 *win_min_max)
1726 {
1727 u32 win_size, min_bit = 0xff, min_winsize = 0xffff, tmp_win_sum = 0;
1728
1729 switch (type) {
1730 case RX_WIN_RD_DQC:
1731 case RX_WIN_TEST_ENG:
1732 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) {
1733 win_size = delay[bit].best_last - delay[bit].best_first;
1734
1735 if (win_size < min_winsize) {
1736 min_bit = bit;
1737 min_winsize = win_size;
1738 }
1739 tmp_win_sum += win_size;
1740 }
1741 dramc_dbg("type:%d vref:%d Min Bit=%d, min_winsize=%d, win sum:%d\n",
1742 type, vref, min_bit, min_winsize, tmp_win_sum);
1743
1744 if (tmp_win_sum > vref_dly->max_win_sum) {
1745 *win_min_max = min_winsize;
1746 vref_dly->max_win_sum = tmp_win_sum;
1747
1748 /* best vref */
1749 vref_dly->best_vref = vref;
1750 }
1751 dramc_dbg("type:%d vref:%d, win_sum_total:%d, tmp_win_sum:%d)\n",
1752 type, vref, vref_dly->max_win_sum, tmp_win_sum);
1753 dramc_set_vref_dly(vref_dly, delay);
1754
1755 if (tmp_win_sum < vref_dly->max_win_sum * 95 / 100) {
1756 dramc_dbg("type:%d best vref found[%d], early break! (%d < %d)\n",
1757 type, vref_dly->best_vref, tmp_win_sum,
1758 vref_dly->max_win_sum * 95 / 100);
1759 return true;
1760 }
1761
1762 break;
1763 case TX_WIN_DQ_ONLY:
1764 case TX_WIN_DQ_DQM:
1765 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) {
1766 win_size = delay[bit].best_last - delay[bit].best_first;
1767
1768 if (win_size < min_winsize) {
1769 min_bit = bit;
1770 min_winsize = win_size;
1771 }
1772 tmp_win_sum += win_size;
1773 }
1774 dramc_dbg("type:%d vref:%d Min Bit=%d, min_winsize=%d, win sum:%d\n",
1775 type, vref, min_bit, min_winsize, tmp_win_sum);
1776
1777 if (min_winsize > *win_min_max ||
1778 (min_winsize == *win_min_max &&
1779 tmp_win_sum > vref_dly->max_win_sum)) {
1780 *win_min_max = min_winsize;
1781 vref_dly->max_win_sum = tmp_win_sum;
1782
1783 /* best vref */
1784 vref_dly->best_vref = vref;
1785 }
1786 dramc_dbg("type:%d vref:%d, win_sum_total:%d, tmp_win_sum:%d)\n",
1787 type, vref, vref_dly->max_win_sum, tmp_win_sum);
1788 dramc_set_vref_dly(vref_dly, delay);
1789
1790 if (tmp_win_sum < vref_dly->max_win_sum * 95 / 100) {
1791 dramc_dbg("type:%d best vref found[%d], early break! (%d < %d)\n",
1792 type, vref_dly->best_vref, tmp_win_sum,
1793 vref_dly->max_win_sum * 95 / 100);
1794 return true;
1795 }
1796
1797 break;
1798
1799 default:
1800 dramc_err("error calibration type: %d\n", type);
1801 break;
1802 }
1803
1804 return false;
1805 }
1806
dramc_set_rx_dqdqs_dly(u8 chn,u8 rank,s32 dly)1807 static void dramc_set_rx_dqdqs_dly(u8 chn, u8 rank, s32 dly)
1808 {
1809 if (dly <= 0) {
1810 /* Set DQS delay */
1811 dramc_set_rx_dly_factor(chn, rank, RX_DQS, -dly);
1812 dram_phy_reset(chn);
1813 } else {
1814 /* Setup time calibration */
1815 dramc_set_rx_dly_factor(chn, rank, RX_DQM, dly);
1816 dram_phy_reset(chn);
1817 dramc_set_rx_dly_factor(chn, rank, RX_DQ, dly);
1818 }
1819 }
1820
dramc_set_tx_best_dly_factor(u8 chn,u8 rank_start,u8 type,struct per_byte_dly * tx_perbyte_dly,u16 * dq_precal_dly,u8 use_delay_cell,u32 * byte_dly_cell)1821 static void dramc_set_tx_best_dly_factor(u8 chn, u8 rank_start, u8 type,
1822 struct per_byte_dly *tx_perbyte_dly, u16 *dq_precal_dly,
1823 u8 use_delay_cell, u32 *byte_dly_cell)
1824 {
1825 u32 dq_large = 0, dq_large_oen = 0, dq_small = 0, dq_small_oen = 0, adjust_center = 1;
1826 u32 dqm_large = 0, dqm_large_oen = 0, dqm_small = 0, dqm_small_oen = 0;
1827 struct tx_dly_tune dqdly_tune[DQS_NUMBER] = {0};
1828 struct tx_dly_tune dqmdly_tune[DQS_NUMBER] = {0};
1829
1830 for (size_t i = 0; i < DQS_NUMBER; i++) {
1831 dramc_transfer_dly_tune(chn, tx_perbyte_dly[i].final_dly,
1832 adjust_center, &dqdly_tune[i]);
1833 dramc_transfer_dly_tune(chn, dq_precal_dly[i],
1834 adjust_center, &dqmdly_tune[i]);
1835
1836 dq_large += dqdly_tune[i].coarse_tune_large << (i * 4);
1837 dq_large_oen += dqdly_tune[i].coarse_tune_large_oen << (i * 4);
1838 dq_small += dqdly_tune[i].coarse_tune_small << (i * 4);
1839 dq_small_oen += dqdly_tune[i].coarse_tune_small_oen << (i * 4);
1840
1841 dqm_large += dqmdly_tune[i].coarse_tune_large << (i * 4);
1842 dqm_large_oen += dqmdly_tune[i].coarse_tune_large_oen << (i * 4);
1843 dqm_small += dqmdly_tune[i].coarse_tune_small << (i * 4);
1844 dqm_small_oen += dqmdly_tune[i].coarse_tune_small_oen << (i * 4);
1845 }
1846
1847 for (size_t rank = rank_start; rank < RANK_MAX; rank++) {
1848 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dq[0],
1849 0x77777777, dq_large | (dq_large_oen << 16));
1850 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dq[2],
1851 0x77777777, dq_small | (dq_small_oen << 16));
1852 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dq[1],
1853 0x77777777, dqm_large | (dqm_large_oen << 16));
1854 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dq[3],
1855 0x77777777, dqm_small | (dqm_small_oen << 16));
1856
1857 for (size_t byte = 0; byte < 2; byte++)
1858 SET32_BITFIELDS(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[7],
1859 FINE_TUNE_DQ, dqdly_tune[byte].fine_tune,
1860 FINE_TUNE_DQM, dqmdly_tune[byte].fine_tune);
1861
1862 if (use_delay_cell == 1) {
1863 for (size_t byte = 0; byte < DQS_NUMBER; byte++)
1864 write32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[0],
1865 byte_dly_cell[byte]);
1866 }
1867
1868 if (type != TX_WIN_DQ_ONLY)
1869 continue;
1870
1871 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].fine_tune, 0x3f3f3f3f,
1872 (dqdly_tune[0].fine_tune << 8) | (dqdly_tune[1].fine_tune << 0) |
1873 (dqmdly_tune[0].fine_tune << 24) | (dqmdly_tune[1].fine_tune << 16));
1874
1875 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal1, 0x7ff | (0x7ff << 16),
1876 (dqdly_tune[0].fine_tune << 0) | (dqdly_tune[1].fine_tune << 16));
1877 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal2, 0x7ff | (0x7ff << 16),
1878 (dqdly_tune[0].fine_tune << 0) | (dqdly_tune[1].fine_tune << 16));
1879 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].dqs2dq_cal5, 0x7ff | (0x7ff << 16),
1880 (dqmdly_tune[0].fine_tune << 0) | (dqmdly_tune[1].fine_tune << 16));
1881 }
1882 }
1883
dramc_set_rx_best_dly_factor(u8 chn,u8 rank,struct win_perbit_dly * dly,s32 * dqsdly_byte,s32 * dqmdly_byte)1884 static void dramc_set_rx_best_dly_factor(u8 chn, u8 rank,
1885 struct win_perbit_dly *dly, s32 *dqsdly_byte, s32 *dqmdly_byte)
1886 {
1887 u32 value;
1888
1889 /* set dqs delay, (dqm delay) */
1890 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
1891 value = (dqsdly_byte[byte] << 24) | (dqsdly_byte[byte] << 16) |
1892 (dqmdly_byte[byte] << 8) | (dqmdly_byte[byte] << 0);
1893 clrsetbits32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[6], 0x7f7f3f3f, value);
1894 }
1895 dram_phy_reset(chn);
1896
1897 /* set dq delay */
1898 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
1899 for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit += 2) {
1900 u8 index = bit + byte * DQS_BIT_NUMBER;
1901 u8 dq_num = 2 + bit / 2;
1902 value = (dly[index + 1].best_dqdly << 24) |
1903 (dly[index + 1].best_dqdly << 16) |
1904 (dly[index].best_dqdly << 8) | (dly[index].best_dqdly << 0);
1905
1906 clrsetbits32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[dq_num],
1907 0x3f3f3f3f, value);
1908 }
1909 }
1910 }
1911
dramc_set_dqdqs_dly(u8 chn,u8 rank,enum CAL_TYPE type,u8 * small_value,s32 dly)1912 static void dramc_set_dqdqs_dly(u8 chn, u8 rank, enum CAL_TYPE type, u8 *small_value, s32 dly)
1913 {
1914 if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG)
1915 dramc_set_rx_dqdqs_dly(chn, rank, dly);
1916 else
1917 dramc_set_tx_dly_factor(chn, rank, type, small_value, dly);
1918 }
1919
dramc_set_tx_dly_center(struct per_byte_dly * center_dly,const struct win_perbit_dly * vref_dly)1920 static void dramc_set_tx_dly_center(struct per_byte_dly *center_dly,
1921 const struct win_perbit_dly *vref_dly)
1922 {
1923 int index;
1924 struct per_byte_dly *dly;
1925
1926 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
1927 dly = ¢er_dly[byte];
1928 dly->min_center = 0xffff;
1929 dly->max_center = 0;
1930
1931 for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) {
1932 index = bit + 8 * byte;
1933 if (vref_dly[index].win_center < dly->min_center)
1934 dly->min_center = vref_dly[index].win_center;
1935 if (vref_dly[index].win_center > dly->max_center)
1936 dly->max_center = vref_dly[index].win_center;
1937 }
1938 dramc_dbg("center_dly[%d].min_center = %d, "
1939 "center_dly[%d].max_center = %d\n",
1940 byte, center_dly[byte].min_center,
1941 byte, center_dly[byte].max_center);
1942 }
1943 }
1944
get_freq_group_clock(u8 freq_group)1945 static u32 get_freq_group_clock(u8 freq_group)
1946 {
1947 u32 clock_rate = 0;
1948
1949 /*
1950 * The clock rate is usually (frequency / 2 - delta), where the delta
1951 * is introduced to avoid interference from RF peripherals like
1952 * modem, WiFi, and Bluetooth.
1953 */
1954 switch (freq_group) {
1955 case LP4X_DDR1600:
1956 clock_rate = 796;
1957 break;
1958 case LP4X_DDR2400:
1959 clock_rate = 1196;
1960 break;
1961 case LP4X_DDR3200:
1962 clock_rate = 1596;
1963 break;
1964 case LP4X_DDR3600:
1965 clock_rate = 1792;
1966 break;
1967 default:
1968 die("Invalid DDR frequency group %u\n", freq_group);
1969 break;
1970 }
1971
1972 return clock_rate;
1973 }
1974
dramc_set_tx_best_dly(u8 chn,u8 rank,bool bypass_tx,struct win_perbit_dly * vref_dly,enum CAL_TYPE type,u8 freq_group,u16 * tx_dq_precal_result,u16 dly_cell_unit,const struct sdram_params * params,const bool fast_calib)1975 static void dramc_set_tx_best_dly(u8 chn, u8 rank, bool bypass_tx,
1976 struct win_perbit_dly *vref_dly,
1977 enum CAL_TYPE type, u8 freq_group,
1978 u16 *tx_dq_precal_result, u16 dly_cell_unit,
1979 const struct sdram_params *params,
1980 const bool fast_calib)
1981 {
1982 int index, clock_rate;
1983 u8 use_delay_cell;
1984 u32 byte_dly_cell[DQS_NUMBER] = { 0 };
1985 struct per_byte_dly center_dly[DQS_NUMBER];
1986 u16 tune_diff, dq_delay_cell[DQ_DATA_WIDTH];
1987
1988 clock_rate = get_freq_group_clock(freq_group);
1989
1990 if (type == TX_WIN_DQ_ONLY && get_freq_fsq(freq_group) == FSP_1)
1991 use_delay_cell = 1;
1992 else
1993 use_delay_cell = 0;
1994
1995 if (fast_calib && bypass_tx) {
1996 dramc_dbg("bypass TX, clock_rate: %d\n", clock_rate);
1997 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
1998 center_dly[byte].min_center = params->tx_center_min[chn][rank][byte];
1999 center_dly[byte].max_center = params->tx_center_max[chn][rank][byte];
2000 for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) {
2001 index = bit + 8 * byte;
2002 vref_dly[index].win_center =
2003 params->tx_win_center[chn][rank][index];
2004 vref_dly[index].best_first =
2005 params->tx_first_pass[chn][rank][index];
2006 vref_dly[index].best_last =
2007 params->tx_last_pass[chn][rank][index];
2008 }
2009 }
2010 } else {
2011 dramc_set_tx_dly_center(center_dly, vref_dly);
2012 }
2013
2014 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
2015 if (use_delay_cell == 0) {
2016 center_dly[byte].final_dly = (center_dly[byte].min_center +
2017 center_dly[byte].max_center) >> 1;
2018 tx_dq_precal_result[byte] = center_dly[byte].final_dly;
2019 } else {
2020 center_dly[byte].final_dly = center_dly[byte].min_center;
2021 tx_dq_precal_result[byte] = (center_dly[byte].min_center
2022 + center_dly[byte].max_center) >> 1;
2023
2024 for (u8 bit = 0; bit < DQS_BIT_NUMBER; bit++) {
2025 index = bit + 8 * byte;
2026 tune_diff = vref_dly[index].win_center -
2027 center_dly[byte].min_center;
2028 dq_delay_cell[index] = ((tune_diff * 100000000) /
2029 (clock_rate * 64)) / dly_cell_unit;
2030 byte_dly_cell[byte] |= (dq_delay_cell[index] << (bit * 4));
2031 dramc_dbg("u1DelayCellOfst[%d]=%d cells (%d PI)\n",
2032 index, dq_delay_cell[index], tune_diff);
2033 }
2034 }
2035 }
2036
2037 dramc_set_tx_best_dly_factor(chn, rank, type, center_dly, tx_dq_precal_result,
2038 use_delay_cell, byte_dly_cell);
2039 }
2040
dramc_set_rx_best_dly(u8 chn,u8 rank,struct win_perbit_dly * perbit_dly)2041 static int dramc_set_rx_best_dly(u8 chn, u8 rank, struct win_perbit_dly *perbit_dly)
2042 {
2043 u8 bit_first, bit_last;
2044 u16 u2TmpDQMSum;
2045 s32 dqsdly_byte[DQS_NUMBER] = {0x0}, dqm_dly_byte[DQS_NUMBER] = {0x0};
2046
2047 for (u8 byte = 0; byte < DQS_NUMBER; byte++) {
2048 u2TmpDQMSum = 0;
2049
2050 bit_first = DQS_BIT_NUMBER * byte;
2051 bit_last = DQS_BIT_NUMBER * byte + DQS_BIT_NUMBER - 1;
2052 dqsdly_byte[byte] = 64;
2053
2054 for (u8 bit = bit_first; bit <= bit_last; bit++) {
2055 if (perbit_dly[bit].win_center < dqsdly_byte[byte])
2056 dqsdly_byte[byte] = perbit_dly[bit].win_center;
2057 }
2058 dqsdly_byte[byte] = (dqsdly_byte[byte] > 0) ? 0 : -dqsdly_byte[byte];
2059
2060 for (u8 bit = bit_first; bit <= bit_last; bit++) {
2061 perbit_dly[bit].best_dqdly = dqsdly_byte[byte] +
2062 perbit_dly[bit].win_center;
2063 u2TmpDQMSum += perbit_dly[bit].best_dqdly;
2064 }
2065
2066 dqm_dly_byte[byte] = u2TmpDQMSum / DQS_BIT_NUMBER;
2067 }
2068
2069 dramc_set_rx_best_dly_factor(chn, rank, perbit_dly, dqsdly_byte, dqm_dly_byte);
2070 return 0;
2071 }
2072
dramc_get_vref_prop(u8 rank,enum CAL_TYPE type,u8 fsp,u8 * vref_scan_en,u8 * vref_begin,u8 * vref_end)2073 static void dramc_get_vref_prop(u8 rank, enum CAL_TYPE type, u8 fsp,
2074 u8 *vref_scan_en, u8 *vref_begin, u8 *vref_end)
2075 {
2076 if (type == RX_WIN_TEST_ENG && rank == RANK_0) {
2077 *vref_scan_en = 1;
2078 if (fsp == FSP_0)
2079 *vref_begin = 0x18;
2080 else
2081 *vref_begin = 0;
2082 *vref_end = RX_VREF_END;
2083 } else if (type == TX_WIN_DQ_ONLY) {
2084 *vref_scan_en = 1;
2085 if (fsp == FSP_0) {
2086 *vref_begin = 27 - 5;
2087 *vref_end = 27 + 5;
2088 } else {
2089 *vref_begin = TX_VREF_BEGIN;
2090 *vref_end = TX_VREF_END;
2091 }
2092 } else {
2093 *vref_scan_en = 0;
2094 *vref_begin = 0;
2095 *vref_end = 1;
2096 }
2097 }
2098
dram_k_perbit(u8 chn,enum CAL_TYPE type)2099 static u32 dram_k_perbit(u8 chn, enum CAL_TYPE type)
2100 {
2101 u32 err_value;
2102
2103 if (type == RX_WIN_RD_DQC) {
2104 err_value = dramc_rx_rd_dqc_run(chn);
2105 } else if (type == RX_WIN_TEST_ENG) {
2106 err_value = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK);
2107 } else {
2108 dramc_engine2_setpat(chn, true);
2109 err_value = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK);
2110 dramc_engine2_setpat(chn, false);
2111 err_value |= dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK);
2112 }
2113 return err_value;
2114 }
2115
dramc_window_perbit_cal_partial(u8 chn,u8 rank,s16 dly_begin,s16 dly_end,s16 dly_step,enum CAL_TYPE type,u8 * small_value,u8 vref_scan_enable,struct win_perbit_dly * win_perbit)2116 static void dramc_window_perbit_cal_partial(u8 chn, u8 rank,
2117 s16 dly_begin, s16 dly_end, s16 dly_step,
2118 enum CAL_TYPE type, u8 *small_value, u8 vref_scan_enable,
2119 struct win_perbit_dly *win_perbit)
2120 {
2121 u32 finish_bit = 0;
2122
2123 for (s16 dly = dly_begin; dly < dly_end; dly += dly_step) {
2124 dramc_set_dqdqs_dly(chn, rank, type, small_value, dly);
2125
2126 u32 err_value = dram_k_perbit(chn, type);
2127 if (!vref_scan_enable)
2128 dramc_dbg("%d ", dly);
2129
2130 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) {
2131 bool bit_fail = (err_value & ((u32)1 << bit)) != 0;
2132
2133 /* pass window bigger than 7,
2134 * consider as real pass window.
2135 */
2136 if (dramc_check_dqdqs_win(&(win_perbit[bit]),
2137 dly, dly_end, bit_fail) > 7)
2138 finish_bit |= (1 << bit);
2139
2140 if (vref_scan_enable)
2141 continue;
2142
2143 dramc_dbg("%s", bit_fail ? "x" : "o");
2144 if (bit % DQS_BIT_NUMBER == 7)
2145 dramc_dbg(" ");
2146 }
2147
2148 if (!vref_scan_enable)
2149 dramc_dbg(" [MSB]\n");
2150
2151 if (finish_bit == 0xffff && (err_value & 0xffff) == 0xffff) {
2152 dramc_dbg("all bits window found, "
2153 "early break! delay=%#x\n", dly);
2154 break;
2155 }
2156 }
2157 }
2158
dramc_window_perbit_cal(u8 chn,u8 rank,u8 freq_group,enum CAL_TYPE type,const struct sdram_params * params,const bool fast_calib)2159 static u8 dramc_window_perbit_cal(u8 chn, u8 rank, u8 freq_group,
2160 enum CAL_TYPE type, const struct sdram_params *params,
2161 const bool fast_calib)
2162 {
2163 u8 vref = 0, vref_begin = 0, vref_end = 1, vref_step = 1, vref_use = 0;
2164 u8 vref_scan_enable = 0, small_reg_value = 0xff;
2165 s16 dly_begin = 0, dly_end = 0, dly_step = 1;
2166 u32 dummy_rd_bak_engine2 = 0, win_min_max = 0;
2167 static u16 dq_precal_result[DQS_NUMBER];
2168 struct vref_perbit_dly vref_dly;
2169 struct win_perbit_dly win_perbit[DQ_DATA_WIDTH];
2170 u16 dly_cell_unit = params->delay_cell_unit;
2171
2172 u8 fsp = get_freq_fsq(freq_group);
2173 u8 vref_range = !fsp;
2174 bool bypass_tx_rx = !fsp;
2175
2176 dramc_dbg("bypass TX RX window: %s\n", bypass_tx_rx ? "Yes" : "No");
2177 dramc_get_vref_prop(rank, type, fsp,
2178 &vref_scan_enable, &vref_begin, &vref_end);
2179 dramc_get_dly_range(chn, rank, type, freq_group, dq_precal_result,
2180 &dly_begin, &dly_end, params);
2181
2182 if (fast_calib) {
2183 if (type == RX_WIN_TEST_ENG && vref_scan_enable == 1) {
2184 vref_begin = params->rx_vref[chn];
2185 vref_end = vref_begin + 1;
2186 dramc_dbg("bypass RX vref: %d\n", vref_begin);
2187 } else if (type == TX_WIN_DQ_ONLY) {
2188 vref_begin = params->tx_vref[chn][rank] | (vref_range << 6);
2189 vref_end = vref_begin + 1;
2190 dramc_dbg("bypass TX vref: %d\n", vref_begin);
2191 }
2192 vref_dly.best_vref = vref_begin;
2193 }
2194
2195 if ((type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) && fsp == FSP_0)
2196 dly_step = 2;
2197
2198 dramc_dbg("[channel %d] [rank %d] type: %d, vref_enable: %d, vref range[%d : %d]\n",
2199 chn, rank, type, vref_scan_enable, vref_begin, vref_end);
2200
2201 if (type == TX_WIN_DQ_ONLY || type == TX_WIN_DQ_DQM) {
2202 for (size_t byte = 0; byte < 2; byte++) {
2203 write32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[0], 0);
2204 clrbits32(&ch[chn].phy.shu[0].rk[rank].b[byte].dq[1],
2205 0xf);
2206 }
2207 setbits32(&ch[chn].phy.misc_ctrl1, 0x1 << 7);
2208 setbits32(&ch[chn].ao.dqsoscr, 0x1 << 7);
2209 if (fsp == FSP_1)
2210 vref_step = 2;
2211 }
2212
2213 if (fast_calib && bypass_tx_rx &&
2214 (type == TX_WIN_DQ_ONLY || type == TX_WIN_DQ_DQM)) {
2215 dramc_set_tx_best_dly(chn, rank, bypass_tx_rx, vref_dly.perbit_dly,
2216 type, freq_group, dq_precal_result, dly_cell_unit,
2217 params, fast_calib);
2218
2219 if (vref_scan_enable)
2220 dramc_set_vref(chn, rank, type, vref_dly.best_vref);
2221 return 0;
2222 }
2223
2224 if (type == RX_WIN_RD_DQC) {
2225 dramc_rx_rd_dqc_init(chn, rank);
2226 } else {
2227 if (type == RX_WIN_TEST_ENG)
2228 dramc_rx_vref_pre_setting(chn);
2229 dummy_rd_bak_engine2 = read32(&ch[chn].ao.dummy_rd);
2230 dramc_engine2_init(chn, rank, TEST2_1_CAL, TEST2_2_CAL, false);
2231 }
2232
2233 vref_dly.max_win_sum = 0;
2234 for (vref = vref_begin; vref < vref_end; vref += vref_step) {
2235 small_reg_value = 0xff;
2236 if (type == TX_WIN_DQ_ONLY)
2237 vref_use = vref | (vref_range << 6);
2238 else
2239 vref_use = vref;
2240
2241 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) {
2242 win_perbit[bit].first_pass = PASS_RANGE_NA;
2243 win_perbit[bit].last_pass = PASS_RANGE_NA;
2244 win_perbit[bit].best_first = PASS_RANGE_NA;
2245 win_perbit[bit].best_last = PASS_RANGE_NA;
2246 }
2247
2248 if (vref_scan_enable)
2249 dramc_set_vref(chn, rank, type, vref_use);
2250
2251 if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG) {
2252 dramc_set_rx_dly_factor(chn, rank,
2253 RX_DQM, FIRST_DQ_DELAY);
2254 dramc_set_rx_dly_factor(chn, rank,
2255 RX_DQ, FIRST_DQ_DELAY);
2256 }
2257
2258 if (fast_calib && bypass_tx_rx &&
2259 (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG)) {
2260 dramc_dbg("bypass RX params\n");
2261 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++) {
2262 win_perbit[bit].best_first =
2263 params->rx_firspass[chn][rank][bit];
2264 win_perbit[bit].best_last =
2265 params->rx_lastpass[chn][rank][bit];
2266 }
2267 } else {
2268 dramc_window_perbit_cal_partial(chn, rank,
2269 dly_begin, dly_end, dly_step,
2270 type, &small_reg_value,
2271 vref_scan_enable, win_perbit);
2272 }
2273
2274 for (size_t bit = 0; bit < DQ_DATA_WIDTH; bit++)
2275 dramc_dbg("Dq[%zd] win width (%d ~ %d) %d\n", bit,
2276 win_perbit[bit].best_first, win_perbit[bit].best_last,
2277 win_perbit[bit].best_last - win_perbit[bit].best_first);
2278
2279 if (dramk_calc_best_vref(type, vref_use, &vref_dly,
2280 win_perbit, &win_min_max))
2281 break;
2282 }
2283
2284 if (type == RX_WIN_RD_DQC)
2285 dramc_rx_rd_dqc_end(chn);
2286 else
2287 dramc_engine2_end(chn, dummy_rd_bak_engine2);
2288
2289 if (vref_scan_enable && type == RX_WIN_TEST_ENG)
2290 dramc_set_vref(chn, rank, type, vref_dly.best_vref);
2291
2292 if (type == RX_WIN_RD_DQC || type == RX_WIN_TEST_ENG)
2293 dramc_set_rx_best_dly(chn, rank, vref_dly.perbit_dly);
2294 else
2295 dramc_set_tx_best_dly(chn, rank, false,
2296 vref_dly.perbit_dly, type, freq_group,
2297 dq_precal_result, dly_cell_unit, params, fast_calib);
2298
2299 if (vref_scan_enable && type == TX_WIN_DQ_ONLY)
2300 dramc_set_vref(chn, rank, type, vref_dly.best_vref);
2301
2302 return 0;
2303 }
2304
dramc_dle_factor_handler(u8 chn,u8 val,u8 freq_group)2305 static void dramc_dle_factor_handler(u8 chn, u8 val, u8 freq_group)
2306 {
2307 u8 start_ext2 = 0, start_ext3 = 0, last_ext2 = 0, last_ext3 = 0;
2308
2309 val = MAX(val, 2);
2310 SET32_BITFIELDS(&ch[chn].ao.shu[0].conf[1],
2311 SHU_CONF1_DATLAT, val,
2312 SHU_CONF1_DATLAT_DSEL, val - 2,
2313 SHU_CONF1_DATLAT_DSEL_PHY, val - 2);
2314
2315 if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600)
2316 start_ext2 = 1;
2317
2318 if (val >= 24)
2319 last_ext2 = last_ext3 = 1;
2320 else if (val >= 18)
2321 last_ext2 = 1;
2322
2323 SET32_BITFIELDS(&ch[chn].ao.shu[0].pipe,
2324 SHU_PIPE_READ_START_EXTEND1, 1,
2325 SHU_PIPE_DLE_LAST_EXTEND1, 1,
2326 SHU_PIPE_READ_START_EXTEND2, start_ext2,
2327 SHU_PIPE_DLE_LAST_EXTEND2, last_ext2,
2328 SHU_PIPE_READ_START_EXTEND3, start_ext3,
2329 SHU_PIPE_DLE_LAST_EXTEND3, last_ext3);
2330 dram_phy_reset(chn);
2331 }
2332
dramc_rx_datlat_cal(u8 chn,u8 rank,u8 freq_group,const struct sdram_params * params,const bool fast_calib,bool * test_passed)2333 static u8 dramc_rx_datlat_cal(u8 chn, u8 rank, u8 freq_group,
2334 const struct sdram_params *params, const bool fast_calib,
2335 bool *test_passed)
2336 {
2337 u32 datlat, begin = 0, first = 0, sum = 0, best_step;
2338 u32 datlat_start = 7;
2339
2340 *test_passed = true;
2341 best_step = READ32_BITFIELD(&ch[chn].ao.shu[0].conf[1], SHU_CONF1_DATLAT);
2342
2343 dramc_dbg("[DATLAT] start. CH%d RK%d DATLAT Default: 0x%x\n",
2344 chn, rank, best_step);
2345
2346 u32 dummy_rd_backup = read32(&ch[chn].ao.dummy_rd);
2347 dramc_engine2_init(chn, rank, TEST2_1_CAL, TEST2_2_CAL, false);
2348
2349 if (fast_calib) {
2350 best_step = params->rx_datlat[chn][rank];
2351 dramc_dbg("bypass DATLAT, best_step: %d\n", best_step);
2352 } else {
2353 for (datlat = datlat_start; datlat < DATLAT_TAP_NUMBER; datlat++) {
2354 dramc_dle_factor_handler(chn, datlat, freq_group);
2355
2356 u32 err = dramc_engine2_run(chn, TE_OP_WRITE_READ_CHECK);
2357 if (err == 0) {
2358 if (begin == 0) {
2359 first = datlat;
2360 begin = 1;
2361 }
2362 if (begin == 1) {
2363 sum++;
2364 if (sum > 4)
2365 break;
2366 }
2367 } else {
2368 if (begin == 1)
2369 begin = 0xff;
2370 }
2371
2372 dramc_dbg("Datlat=%2d, err_value=0x%4x, sum=%d\n", datlat, err, sum);
2373 }
2374
2375 dramc_engine2_end(chn, dummy_rd_backup);
2376
2377 *test_passed = (sum != 0);
2378 if (!*test_passed) {
2379 dramc_err("DRAM memory test failed\n");
2380 return 0;
2381 }
2382
2383 if (sum <= 3)
2384 best_step = first + (sum >> 1);
2385 else
2386 best_step = first + 2;
2387 dramc_dbg("First_step=%d, total pass=%d, best_step=%d\n",
2388 begin, sum, best_step);
2389 }
2390
2391 dramc_dle_factor_handler(chn, best_step, freq_group);
2392
2393 SET32_BITFIELDS(&ch[chn].ao.padctrl,
2394 PADCTRL_DQIENQKEND, 1, PADCTRL_DQIENLATEBEGIN, 1);
2395
2396 return (u8)best_step;
2397 }
2398
dramc_dual_rank_rx_datlat_cal(u8 chn,u8 freq_group,u8 datlat0,u8 datlat1)2399 static void dramc_dual_rank_rx_datlat_cal(u8 chn, u8 freq_group, u8 datlat0, u8 datlat1)
2400 {
2401 u8 final_datlat = MAX(datlat0, datlat1);
2402 dramc_dle_factor_handler(chn, final_datlat, freq_group);
2403 }
2404
dramc_rx_dqs_gating_post_process(u8 chn,u8 freq_group,u32 rk_num)2405 static void dramc_rx_dqs_gating_post_process(u8 chn, u8 freq_group, u32 rk_num)
2406 {
2407 s8 dqsinctl;
2408 u32 read_dqsinctl, rankinctl_root, reg_tx_dly_dqsgated_min = 3;
2409 u8 txdly_cal_min = 0xff, txdly_cal_max = 0, tx_dly_dqs_gated = 0;
2410 u32 best_coarse_tune2t[RANK_MAX][DQS_NUMBER];
2411 u32 best_coarse_tune2t_p1[RANK_MAX][DQS_NUMBER];
2412
2413 if (freq_group == LP4X_DDR3200 || freq_group == LP4X_DDR3600)
2414 reg_tx_dly_dqsgated_min = 2;
2415 else
2416 reg_tx_dly_dqsgated_min = 1;
2417
2418 /* get TXDLY_Cal_min and TXDLY_Cal_max value */
2419 for (size_t rank = 0; rank < rk_num; rank++) {
2420 u32 dqsg0 = read32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0);
2421 for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) {
2422 best_coarse_tune2t[rank][dqs] = (dqsg0 >> (dqs * 8)) & 0x7;
2423 best_coarse_tune2t_p1[rank][dqs] = (dqsg0 >> (dqs * 8 + 4)) & 0x7;
2424 dramc_dbg("Rank%zd best DQS%zd dly(2T,(P1)2T)=(%d, %d)\n",
2425 rank, dqs, best_coarse_tune2t[rank][dqs],
2426 best_coarse_tune2t_p1[rank][dqs]);
2427
2428 tx_dly_dqs_gated = best_coarse_tune2t[rank][dqs];
2429 txdly_cal_min = MIN(txdly_cal_min, tx_dly_dqs_gated);
2430
2431 tx_dly_dqs_gated = best_coarse_tune2t_p1[rank][dqs];
2432 txdly_cal_max = MAX(txdly_cal_max, tx_dly_dqs_gated);
2433 }
2434 }
2435
2436 dqsinctl = reg_tx_dly_dqsgated_min - txdly_cal_min;
2437 dramc_dbg("Dqsinctl:%d, dqsgated_min %d, txdly_cal_min %d, txdly_cal_max %d\n",
2438 dqsinctl, reg_tx_dly_dqsgated_min, txdly_cal_min, txdly_cal_max);
2439
2440 if (dqsinctl != 0) {
2441 txdly_cal_min += dqsinctl;
2442 txdly_cal_max += dqsinctl;
2443
2444 for (size_t rank = 0; rank < rk_num; rank++) {
2445 dramc_dbg("Rank: %zd\n", rank);
2446 for (size_t dqs = 0; dqs < DQS_NUMBER; dqs++) {
2447 best_coarse_tune2t[rank][dqs] += dqsinctl;
2448 best_coarse_tune2t_p1[rank][dqs] += dqsinctl;
2449
2450 dramc_dbg("Best DQS%zd dly(2T) = (%d)\n",
2451 dqs, best_coarse_tune2t[rank][dqs]);
2452 dramc_dbg("Best DQS%zd P1 dly(2T) = (%d)\n",
2453 dqs, best_coarse_tune2t_p1[rank][dqs]);
2454 }
2455
2456 clrsetbits32(&ch[chn].ao.shu[0].rk[rank].selph_dqsg0,
2457 0x77777777,
2458 (best_coarse_tune2t[rank][0] << 0) |
2459 (best_coarse_tune2t[rank][1] << 8) |
2460 (best_coarse_tune2t_p1[rank][0] << 4) |
2461 (best_coarse_tune2t_p1[rank][1] << 12));
2462 }
2463 }
2464
2465 read_dqsinctl = READ32_BITFIELD(&ch[chn].ao.shu[0].rk[0].dqsctl,
2466 SHURK_DQSCTL_DQSINCTL) - dqsinctl;
2467 rankinctl_root = (read_dqsinctl >= 2) ? (read_dqsinctl - 2) : 0;
2468
2469 SET32_BITFIELDS(&ch[chn].ao.shu[0].rk[0].dqsctl, SHURK_DQSCTL_DQSINCTL, read_dqsinctl);
2470 SET32_BITFIELDS(&ch[chn].ao.shu[0].rk[1].dqsctl, SHURK_DQSCTL_DQSINCTL, read_dqsinctl);
2471 clrsetbits32(&ch[chn].ao.shu[0].rankctl,
2472 (0xf << 28) | (0xf << 20) | (0xf << 24) | 0xf,
2473 (read_dqsinctl << 28) | (rankinctl_root << 20) |
2474 (rankinctl_root << 24) | rankinctl_root);
2475
2476 u8 ROEN = read32(&ch[chn].ao.shu[0].odtctrl) & 0x1;
2477 clrsetbits32(&ch[chn].ao.shu[0].rodtenstb, (0xffff << 8) | (0x3f << 2) | (0x1),
2478 (0xff << 8) | (0x9 << 2) | ROEN);
2479 }
2480
start_dqsosc(u8 chn)2481 static void start_dqsosc(u8 chn)
2482 {
2483 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSOSCENEN, 1);
2484 if (!wait_us(100, READ32_BITFIELD(&ch[chn].nao.spcmdresp,
2485 SPCMDRESP_DQSOSCEN_RESPONSE))) {
2486 dramc_err("start dqsosc timed out\n");
2487 return;
2488 }
2489 SET32_BITFIELDS(&ch[chn].ao.spcmd, SPCMD_DQSOSCENEN, 0);
2490 }
2491
dqsosc_auto(u8 chn,u8 rank,u8 freq_group,u16 * osc_thrd_inc,u16 * osc_thrd_dec)2492 static void dqsosc_auto(u8 chn, u8 rank, u8 freq_group,
2493 u16 *osc_thrd_inc, u16 *osc_thrd_dec)
2494 {
2495 u8 mr23 = MR23_DEFAULT_VALUE;
2496 u16 mr18, mr19;
2497 u16 dqsosc_cnt[2], dqs_cnt, dqsosc, thrd_inc, thrd_dec;
2498 u32 clock_rate, tck;
2499
2500 struct reg_value regs_bak[] = {
2501 {&ch[chn].ao.mrs},
2502 {&ch[chn].ao.dramc_pd_ctrl},
2503 {&ch[chn].ao.ckectrl},
2504 };
2505
2506 for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++)
2507 regs_bak[i].value = read32(regs_bak[i].addr);
2508
2509 SET32_BITFIELDS(&ch[chn].ao.rkcfg, RKCFG_DQSOSC2RK, 0);
2510 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, rank);
2511 SET32_BITFIELDS(&ch[chn].ao.mpc_option, MPC_OPTION_MPCRKEN, 1);
2512
2513 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRSRK, rank);
2514 dramc_mode_reg_write(chn, 23, mr23);
2515
2516 for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++)
2517 SET32_BITFIELDS(&ch[chn].ao.shu[shu].scintv,
2518 SHU_SCINTV_DQSOSCENDIS, 1);
2519
2520 SET32_BITFIELDS(&ch[chn].ao.dramc_pd_ctrl,
2521 DRAMC_PD_CTRL_MIOCKCTRLOFF, 1);
2522 dramc_cke_fix_onoff(CKE_FIXON, chn);
2523
2524 start_dqsosc(chn);
2525 udelay(1);
2526 SET32_BITFIELDS(&ch[chn].ao.mrs, MRS_MRRRK, rank);
2527
2528 mr18 = dramc_mode_reg_read(chn, 18);
2529 mr19 = dramc_mode_reg_read(chn, 19);
2530 dqsosc_cnt[0] = (mr18 & 0xff) | ((mr19 & 0xff) << 8);
2531 dqsosc_cnt[1] = (mr18 >> 8) | (mr19 & 0xff00);
2532 dramc_dbg("DQSOscCnt B0=%#x, B1=%#x\n", dqsosc_cnt[0], dqsosc_cnt[1]);
2533
2534 /* get the INC and DEC values */
2535 clock_rate = get_freq_group_clock(freq_group);
2536 tck = 1000000 / clock_rate;
2537
2538 dqs_cnt = (mr18 & 0xff) | ((mr19 & 0xff) << 8);
2539 if (dqs_cnt != 0) {
2540 dqsosc = mr23 * 16 * 1000000 / (2 * dqs_cnt * clock_rate);
2541 thrd_inc = mr23 * tck * tck / (dqsosc * dqsosc * 10);
2542 thrd_dec = 3 * mr23 * tck * tck / (dqsosc * dqsosc * 20);
2543 } else {
2544 dqsosc = 0;
2545 thrd_inc = 0x6;
2546 thrd_dec = 0x4;
2547 }
2548 osc_thrd_inc[rank] = thrd_inc;
2549 osc_thrd_dec[rank] = thrd_dec;
2550 dramc_dbg("CH%d_RK%d: MR18=%#x, MR19=%#x, DQSOSC=%d, MR23=%d, "
2551 "INC=%d, DEC=%d\n",
2552 chn, rank, mr18, mr19, dqsosc, mr23, thrd_inc, thrd_dec);
2553
2554 for (size_t i = 0; i < ARRAY_SIZE(regs_bak); i++)
2555 write32(regs_bak[i].addr, regs_bak[i].value);
2556
2557 SET32_BITFIELDS(&ch[chn].ao.shu[0].rk[rank].dqsosc,
2558 SHU1RK0_DQSOSC_DQSOSC_BASE_RK0, dqsosc_cnt[0],
2559 SHU1RK0_DQSOSC_DQSOSC_BASE_RK0_B1, dqsosc_cnt[1]);
2560 }
2561
dramc_hw_dqsosc(u8 chn,u32 rk_num)2562 void dramc_hw_dqsosc(u8 chn, u32 rk_num)
2563 {
2564 u32 freq_shu1 = get_shu_freq(DRAM_DFS_SHUFFLE_1);
2565 u32 freq_shu2 = get_shu_freq(DRAM_DFS_SHUFFLE_2);
2566 u32 freq_shu3 = get_shu_freq(DRAM_DFS_SHUFFLE_3);
2567
2568 SET32_BITFIELDS(&ch[chn].ao.rk[2].dqsosc,
2569 RK2_DQSOSC_FREQ_RATIO_TX_0, freq_shu2 * 8 / freq_shu1,
2570 RK2_DQSOSC_FREQ_RATIO_TX_1, freq_shu3 * 8 / freq_shu1);
2571 SET32_BITFIELDS(&ch[chn].ao.rk[2].dqsosc,
2572 RK2_DQSOSC_FREQ_RATIO_TX_3, freq_shu1 * 8 / freq_shu2,
2573 RK2_DQSOSC_FREQ_RATIO_TX_4, freq_shu3 * 8 / freq_shu2);
2574 SET32_BITFIELDS(&ch[chn].ao.rk[2].dummy_rd_bk,
2575 RK2_DUMMY_RD_BK_FREQ_RATIO_TX_6,
2576 freq_shu1 * 8 / freq_shu3,
2577 RK2_DUMMY_RD_BK_FREQ_RATIO_TX_7,
2578 freq_shu2 * 8 / freq_shu3);
2579
2580 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[0],
2581 PRE_TDQSCK1_SHU_PRELOAD_TX_HW, 1,
2582 PRE_TDQSCK1_SHU_PRELOAD_TX_START, 0,
2583 PRE_TDQSCK1_SW_UP_TX_NOW_CASE, 0);
2584
2585 SET32_BITFIELDS(&ch[chn].ao.mpc_option, MPC_OPTION_MPC_BLOCKALE_OPT, 0);
2586 SET32_BITFIELDS(&ch[chn].phy.misc_ctrl1, MISC_CTRL1_R_DMARPIDQ_SW, 1);
2587 SET32_BITFIELDS(&ch[chn].ao.dqsoscr, DQSOSCR_ARUIDQ_SW, 1);
2588 SET32_BITFIELDS(&ch[chn].ao.dqsoscr, DQSOSCR_DQSOSCRDIS, 1);
2589
2590 SET32_BITFIELDS(&ch[chn].ao.rk[0].dqsosc, RK0_DQSOSC_DQSOSCR_RK0EN, 1);
2591
2592 if (rk_num == RANK_MAX)
2593 SET32_BITFIELDS(&ch[chn].ao.rk[1].dqsosc, RK1_DQSOSC_DQSOSCR_RK1EN, 1);
2594
2595 SET32_BITFIELDS(&ch[chn].ao.dqsoscr, DQSOSCR_DQSOSC_CALEN, 1);
2596
2597 for (u8 shu = 0; shu < DRAM_DFS_SHUFFLE_MAX; shu++)
2598 SET32_BITFIELDS(&ch[chn].ao.shu[shu].scintv,
2599 SHU_SCINTV_DQSOSCENDIS, 1);
2600 }
2601
dqsosc_shu_settings(u8 chn,u8 freq_group,u16 * osc_thrd_inc,u16 * osc_thrd_dec)2602 static void dqsosc_shu_settings(u8 chn, u8 freq_group,
2603 u16 *osc_thrd_inc, u16 *osc_thrd_dec)
2604 {
2605 u8 filt_pithrd, w2r_sel, upd_sel;
2606 u8 mr23 = MR23_DEFAULT_VALUE;
2607 u16 prd_cnt, thrd_inc, thrd_dec;
2608
2609 SET32_BITFIELDS(&ch[chn].ao.shu[0].scintv,
2610 SHU_SCINTV_DQS2DQ_SHU_PITHRD, 0);
2611 SET32_BITFIELDS(&ch[chn].ao.rk[0].dqsosc,
2612 RK0_DQSOSC_R_DMDQS2DQ_FILT_OPT, 0);
2613
2614 switch (freq_group) {
2615 case LP4X_DDR1600:
2616 filt_pithrd = 0x5;
2617 w2r_sel = 0x5;
2618 upd_sel = 0x0;
2619 break;
2620 case LP4X_DDR2400:
2621 filt_pithrd = 0x8;
2622 w2r_sel = 0x2;
2623 upd_sel = 0x0;
2624 break;
2625 case LP4X_DDR3200:
2626 filt_pithrd = 0xA;
2627 w2r_sel = 0x2;
2628 upd_sel = 0x0;
2629 break;
2630 case LP4X_DDR3600:
2631 filt_pithrd = 0xB;
2632 w2r_sel = 0x2;
2633 upd_sel = 0x0;
2634 break;
2635 default:
2636 die("Invalid DDR frequency group %u\n", freq_group);
2637 return;
2638 }
2639
2640 SET32_BITFIELDS(&ch[chn].ao.shu[0].scintv,
2641 SHU_SCINTV_DQS2DQ_FILT_PITHRD, filt_pithrd);
2642 SET32_BITFIELDS(&ch[chn].ao.shu[0].wodt,
2643 SHU1_WODT_TXUPD_W2R_SEL, w2r_sel,
2644 SHU1_WODT_TXUPD_SEL, upd_sel);
2645
2646 prd_cnt = mr23 / 4 + 3;
2647 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsosc_prd,
2648 SHU1_DQSOSC_PRD_DQSOSC_PRDCNT, prd_cnt);
2649 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsoscr,
2650 SHU_DQSOSCR_DQSOSCRCNT, 0x40);
2651
2652 for (u8 rk = RANK_0; rk < RANK_MAX; rk++) {
2653 thrd_inc = osc_thrd_inc[rk];
2654 thrd_dec = osc_thrd_dec[rk];
2655
2656 if (rk == RANK_0) {
2657 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsoscthrd,
2658 SHU_DQSOSCTHRD_DQSOSCTHRD_INC_RK0,
2659 thrd_inc);
2660 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsoscthrd,
2661 SHU_DQSOSCTHRD_DQSOSCTHRD_DEC_RK0,
2662 thrd_dec);
2663 } else {
2664 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsoscthrd,
2665 SHU_DQSOSCTHRD_DQSOSCTHRD_INC_RK1_7TO0,
2666 thrd_inc & 0xFF);
2667 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsosc_prd,
2668 SHU1_DQSOSC_PRD_DQSOSCTHRD_INC_RK1_11TO8,
2669 (thrd_inc & 0xF00) >> 8);
2670 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsosc_prd,
2671 SHU1_DQSOSC_PRD_DQSOSCTHRD_DEC_RK1,
2672 thrd_dec);
2673 }
2674 }
2675
2676 SET32_BITFIELDS(&ch[chn].ao.shu[0].dqsoscr2,
2677 SHU_DQSOSCR2_DQSOSCENCNT, 0x1FF);
2678 }
2679
dramc_dqs_precalculation_preset(void)2680 void dramc_dqs_precalculation_preset(void)
2681 {
2682 u32 jump_ratio_index = 0;
2683 u16 jump_ratio[DRAM_DFS_SHUFFLE_MAX * HW_REG_SHUFFLE_MAX] = {0};
2684 u32 u4value = 0, u4value1 = 0;
2685
2686 for (u8 shu_src = 0; shu_src < HW_REG_SHUFFLE_MAX; shu_src++)
2687 for (u8 shu_dst = 0; shu_dst < HW_REG_SHUFFLE_MAX; shu_dst++) {
2688 if (shu_src == shu_dst)
2689 continue;
2690 if (shu_src >= DRAM_DFS_SHUFFLE_MAX ||
2691 shu_dst >= DRAM_DFS_SHUFFLE_MAX) {
2692 jump_ratio_index++;
2693 continue;
2694 }
2695
2696 jump_ratio[jump_ratio_index] = DIV_ROUND_CLOSEST(
2697 (get_shu_freq(shu_dst) >> 1) * 32,
2698 get_shu_freq(shu_src) >> 1);
2699 dramc_dbg("Jump_RATIO [%d]: %x Freq %d -> %d DDR%d ->"
2700 " DDR%d\n",
2701 jump_ratio_index,
2702 jump_ratio[jump_ratio_index],
2703 shu_src + 1, shu_dst + 1,
2704 get_shu_freq(shu_src), get_shu_freq(shu_dst));
2705 jump_ratio_index++;
2706 }
2707
2708 for (size_t chn = 0; chn < CHANNEL_MAX; chn++) {
2709 struct dramc_ao_regs_shu *shu = &ch[chn].ao.shu[0];
2710 struct dramc_ao_regs_rk *rk = &ch[chn].ao.rk[0];
2711 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[0],
2712 PRE_TDQSCK1_TDQSCK_PRECAL_HW, 1);
2713 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[1],
2714 PRE_TDQSCK2_TDDQSCK_JUMP_RATIO0, jump_ratio[0],
2715 PRE_TDQSCK2_TDDQSCK_JUMP_RATIO1, jump_ratio[1],
2716 PRE_TDQSCK2_TDDQSCK_JUMP_RATIO2, jump_ratio[2],
2717 PRE_TDQSCK2_TDDQSCK_JUMP_RATIO3, jump_ratio[3]);
2718 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[2],
2719 PRE_TDQSCK3_TDDQSCK_JUMP_RATIO4, jump_ratio[4],
2720 PRE_TDQSCK3_TDDQSCK_JUMP_RATIO5, jump_ratio[5],
2721 PRE_TDQSCK3_TDDQSCK_JUMP_RATIO6, jump_ratio[6],
2722 PRE_TDQSCK3_TDDQSCK_JUMP_RATIO7, jump_ratio[7]);
2723 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[3],
2724 PRE_TDQSCK4_TDDQSCK_JUMP_RATIO8, jump_ratio[8],
2725 PRE_TDQSCK4_TDDQSCK_JUMP_RATIO9, jump_ratio[9],
2726 PRE_TDQSCK4_TDDQSCK_JUMP_RATIO10, jump_ratio[10],
2727 PRE_TDQSCK4_TDDQSCK_JUMP_RATIO11, jump_ratio[11]);
2728
2729 for (u8 rnk = RANK_0; rnk < RANK_MAX; rnk++) {
2730 /* Shuffle 0 */
2731 u4value = READ32_BITFIELD(
2732 &shu[0].rk[rnk].selph_dqsg0,
2733 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED);
2734 u4value1 = READ32_BITFIELD(
2735 &shu[0].rk[rnk].selph_dqsg1,
2736 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED);
2737 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[0],
2738 RK0_PRE_TDQSCK1_TDQSCK_UIFREQ1_B0R0,
2739 (u4value << 3) | u4value1);
2740 u4value = READ32_BITFIELD(
2741 &shu[0].rk[rnk].dqsien,
2742 SHURK0_DQSIEN_R0DQS0IEN);
2743 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[0],
2744 RK0_PRE_TDQSCK1_TDQSCK_PIFREQ1_B0R0,
2745 u4value);
2746 u4value = READ32_BITFIELD(
2747 &shu[0].rk[rnk].selph_dqsg0,
2748 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED_P1);
2749 u4value1 = READ32_BITFIELD(
2750 &shu[0].rk[rnk].selph_dqsg1,
2751 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED_P1);
2752 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[2],
2753 RK0_PRE_TDQSCK3_TDQSCK_UIFREQ1_P1_B0R0,
2754 (u4value << 3) | u4value1);
2755 /* Shuffle 1 */
2756 u4value = READ32_BITFIELD(
2757 &shu[1].rk[rnk].selph_dqsg0,
2758 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED);
2759 u4value1 = READ32_BITFIELD(
2760 &shu[1].rk[rnk].selph_dqsg1,
2761 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED);
2762 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[0],
2763 RK0_PRE_TDQSCK1_TDQSCK_UIFREQ2_B0R0,
2764 (u4value << 3) | u4value1);
2765 u4value = READ32_BITFIELD(
2766 &shu[1].rk[rnk].dqsien,
2767 SHURK0_DQSIEN_R0DQS0IEN);
2768 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[0],
2769 RK0_PRE_TDQSCK1_TDQSCK_PIFREQ2_B0R0, u4value);
2770 u4value = READ32_BITFIELD(
2771 &shu[1].rk[rnk].selph_dqsg0,
2772 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED_P1);
2773 u4value1 = READ32_BITFIELD(
2774 &shu[1].rk[rnk].selph_dqsg1,
2775 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED_P1);
2776 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[2],
2777 RK0_PRE_TDQSCK3_TDQSCK_UIFREQ2_P1_B0R0,
2778 (u4value << 3) | u4value1);
2779 /* Shuffle 2 */
2780 u4value = READ32_BITFIELD(
2781 &shu[2].rk[rnk].selph_dqsg0,
2782 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED);
2783 u4value1 = READ32_BITFIELD(
2784 &shu[2].rk[rnk].selph_dqsg1,
2785 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED);
2786 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[1],
2787 RK0_PRE_TDQSCK2_TDQSCK_UIFREQ3_B0R0,
2788 (u4value << 3) | u4value1);
2789 u4value = READ32_BITFIELD(
2790 &shu[2].rk[rnk].dqsien,
2791 SHURK0_DQSIEN_R0DQS0IEN);
2792 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[1],
2793 RK0_PRE_TDQSCK2_TDQSCK_PIFREQ3_B0R0,
2794 u4value);
2795 u4value = READ32_BITFIELD(
2796 &shu[2].rk[rnk].selph_dqsg0,
2797 SHURK0_SELPH_DQSG0_TX_DLY_DQS0_GATED_P1);
2798 u4value1 = READ32_BITFIELD(
2799 &shu[2].rk[rnk].selph_dqsg1,
2800 SHURK0_SELPH_DQSG1_REG_DLY_DQS0_GATED_P1);
2801 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[2],
2802 RK0_PRE_TDQSCK3_TDQSCK_UIFREQ3_P1_B0R0,
2803 (u4value << 3) | u4value1);
2804
2805 /* Byte 1 */
2806 /* Shuffle 0 */
2807 u4value = READ32_BITFIELD(
2808 &shu[0].rk[rnk].selph_dqsg0,
2809 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED);
2810 u4value1 = READ32_BITFIELD(
2811 &shu[0].rk[rnk].selph_dqsg1,
2812 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED);
2813 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[3],
2814 RK0_PRE_TDQSCK4_TDQSCK_UIFREQ1_B1R0,
2815 (u4value << 3) | u4value1);
2816 u4value = READ32_BITFIELD(
2817 &shu[0].rk[rnk].dqsien,
2818 SHURK0_DQSIEN_R0DQS1IEN);
2819 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[3],
2820 RK0_PRE_TDQSCK4_TDQSCK_PIFREQ1_B1R0,
2821 u4value);
2822 u4value = READ32_BITFIELD(
2823 &shu[0].rk[rnk].selph_dqsg0,
2824 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED_P1);
2825 u4value1 = READ32_BITFIELD(
2826 &shu[0].rk[rnk].selph_dqsg1,
2827 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED_P1);
2828 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[5],
2829 RK0_PRE_TDQSCK6_TDQSCK_UIFREQ1_P1_B1R0,
2830 (u4value << 3) | u4value1);
2831 /* Shuffle 1 */
2832 u4value = READ32_BITFIELD(
2833 &shu[1].rk[rnk].selph_dqsg0,
2834 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED);
2835 u4value1 = READ32_BITFIELD(
2836 &shu[1].rk[rnk].selph_dqsg1,
2837 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED);
2838 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[3],
2839 RK0_PRE_TDQSCK4_TDQSCK_UIFREQ2_B1R0,
2840 (u4value << 3) | u4value1);
2841 u4value = READ32_BITFIELD(
2842 &shu[1].rk[rnk].dqsien,
2843 SHURK0_DQSIEN_R0DQS1IEN);
2844 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[3],
2845 RK0_PRE_TDQSCK4_TDQSCK_PIFREQ2_B1R0,
2846 u4value);
2847 u4value = READ32_BITFIELD(
2848 &shu[1].rk[rnk].selph_dqsg0,
2849 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED_P1);
2850 u4value1 = READ32_BITFIELD(
2851 &shu[1].rk[rnk].selph_dqsg1,
2852 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED_P1);
2853 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[5],
2854 RK0_PRE_TDQSCK6_TDQSCK_UIFREQ2_P1_B1R0,
2855 (u4value << 3) | u4value1);
2856 /* Shuffle 2 */
2857 u4value = READ32_BITFIELD(
2858 &shu[2].rk[rnk].selph_dqsg0,
2859 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED);
2860 u4value1 = READ32_BITFIELD(
2861 &shu[2].rk[rnk].selph_dqsg1,
2862 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED);
2863 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[4],
2864 RK0_PRE_TDQSCK5_TDQSCK_UIFREQ3_B1R0,
2865 (u4value << 3) | u4value1);
2866 u4value = READ32_BITFIELD(
2867 &shu[2].rk[rnk].dqsien,
2868 SHURK0_DQSIEN_R0DQS1IEN);
2869 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[4],
2870 RK0_PRE_TDQSCK5_TDQSCK_PIFREQ3_B1R0,
2871 u4value);
2872 u4value = READ32_BITFIELD(
2873 &shu[2].rk[rnk].selph_dqsg0,
2874 SHURK0_SELPH_DQSG0_TX_DLY_DQS1_GATED_P1);
2875 u4value1 = READ32_BITFIELD(
2876 &shu[2].rk[rnk].selph_dqsg1,
2877 SHURK0_SELPH_DQSG1_REG_DLY_DQS1_GATED_P1);
2878 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[5],
2879 RK0_PRE_TDQSCK6_TDQSCK_UIFREQ3_P1_B1R0,
2880 (u4value << 3) | u4value1);
2881
2882 /* Byte 2 */
2883 /* Shuffle 0 */
2884 u4value = READ32_BITFIELD(
2885 &shu[0].rk[rnk].selph_dqsg0,
2886 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED);
2887 u4value1 = READ32_BITFIELD(
2888 &shu[0].rk[rnk].selph_dqsg1,
2889 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED);
2890 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[6],
2891 RK0_PRE_TDQSCK7_TDQSCK_UIFREQ1_B2R0,
2892 (u4value << 3) | u4value1);
2893 u4value = READ32_BITFIELD(
2894 &shu[0].rk[rnk].dqsien,
2895 SHURK0_DQSIEN_R0DQS2IEN);
2896 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[6],
2897 RK0_PRE_TDQSCK7_TDQSCK_PIFREQ1_B2R0,
2898 u4value);
2899 u4value = READ32_BITFIELD(
2900 &shu[0].rk[rnk].selph_dqsg0,
2901 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED_P1);
2902 u4value1 = READ32_BITFIELD(
2903 &shu[0].rk[rnk].selph_dqsg1,
2904 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED_P1);
2905 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[8],
2906 RK0_PRE_TDQSCK9_TDQSCK_UIFREQ1_P1_B2R0,
2907 (u4value << 3) | u4value1);
2908 /* Shuffle 1 */
2909 u4value = READ32_BITFIELD(
2910 &shu[1].rk[rnk].selph_dqsg0,
2911 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED);
2912 u4value1 = READ32_BITFIELD(
2913 &shu[1].rk[rnk].selph_dqsg1,
2914 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED);
2915 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[6],
2916 RK0_PRE_TDQSCK7_TDQSCK_UIFREQ2_B2R0,
2917 (u4value << 3) | u4value1);
2918 u4value = READ32_BITFIELD(
2919 &shu[1].rk[rnk].dqsien,
2920 SHURK0_DQSIEN_R0DQS2IEN);
2921 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[6],
2922 RK0_PRE_TDQSCK7_TDQSCK_PIFREQ2_B2R0,
2923 u4value);
2924 u4value = READ32_BITFIELD(
2925 &shu[1].rk[rnk].selph_dqsg0,
2926 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED_P1);
2927 u4value1 = READ32_BITFIELD(
2928 &shu[1].rk[rnk].selph_dqsg1,
2929 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED_P1);
2930 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[8],
2931 RK0_PRE_TDQSCK9_TDQSCK_UIFREQ2_P1_B2R0,
2932 (u4value << 3) | u4value1);
2933 /* Shuffle 2 */
2934 u4value = READ32_BITFIELD(
2935 &shu[2].rk[rnk].selph_dqsg0,
2936 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED);
2937 u4value1 = READ32_BITFIELD(
2938 &shu[2].rk[rnk].selph_dqsg1,
2939 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED);
2940 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[7],
2941 RK0_PRE_TDQSCK8_TDQSCK_UIFREQ3_B2R0,
2942 (u4value << 3) | u4value1);
2943 u4value = READ32_BITFIELD(
2944 &shu[2].rk[rnk].dqsien,
2945 SHURK0_DQSIEN_R0DQS2IEN);
2946 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[7],
2947 RK0_PRE_TDQSCK8_TDQSCK_PIFREQ3_B2R0,
2948 u4value);
2949 u4value = READ32_BITFIELD(
2950 &shu[2].rk[rnk].selph_dqsg0,
2951 SHURK0_SELPH_DQSG0_TX_DLY_DQS2_GATED_P1);
2952 u4value1 = READ32_BITFIELD(
2953 &shu[2].rk[rnk].selph_dqsg1,
2954 SHURK0_SELPH_DQSG1_REG_DLY_DQS2_GATED_P1);
2955 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[8],
2956 RK0_PRE_TDQSCK9_TDQSCK_UIFREQ3_P1_B2R0,
2957 (u4value << 3) | u4value1);
2958
2959 /* Byte 3 */
2960 /* Shuffle 0 */
2961 u4value = READ32_BITFIELD(
2962 &shu[0].rk[rnk].selph_dqsg0,
2963 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED);
2964 u4value1 = READ32_BITFIELD(
2965 &shu[0].rk[rnk].selph_dqsg1,
2966 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED);
2967 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[9],
2968 RK0_PRE_TDQSCK10_TDQSCK_UIFREQ1_B3R0,
2969 (u4value << 3) | u4value1);
2970 u4value = READ32_BITFIELD(
2971 &shu[0].rk[rnk].dqsien,
2972 SHURK0_DQSIEN_R0DQS3IEN);
2973 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[9],
2974 RK0_PRE_TDQSCK10_TDQSCK_PIFREQ1_B3R0,
2975 u4value);
2976 u4value = READ32_BITFIELD(
2977 &shu[0].rk[rnk].selph_dqsg0,
2978 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED_P1);
2979 u4value1 = READ32_BITFIELD(
2980 &shu[0].rk[rnk].selph_dqsg1,
2981 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED_P1);
2982 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[11],
2983 RK0_PRE_TDQSCK12_TDQSCK_UIFREQ1_P1_B3R0,
2984 (u4value << 3) | u4value1);
2985 /* Shuffle 1 */
2986 u4value = READ32_BITFIELD(
2987 &shu[1].rk[rnk].selph_dqsg0,
2988 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED);
2989 u4value1 = READ32_BITFIELD(
2990 &shu[1].rk[rnk].selph_dqsg1,
2991 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED);
2992 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[9],
2993 RK0_PRE_TDQSCK10_TDQSCK_UIFREQ2_B3R0,
2994 (u4value << 3) | u4value1);
2995 u4value = READ32_BITFIELD(
2996 &shu[1].rk[rnk].dqsien,
2997 SHURK0_DQSIEN_R0DQS3IEN);
2998 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[9],
2999 RK0_PRE_TDQSCK10_TDQSCK_PIFREQ2_B3R0,
3000 u4value);
3001 u4value = READ32_BITFIELD(
3002 &shu[1].rk[rnk].selph_dqsg0,
3003 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED_P1);
3004 u4value1 = READ32_BITFIELD(
3005 &shu[1].rk[rnk].selph_dqsg1,
3006 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED_P1);
3007 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[11],
3008 RK0_PRE_TDQSCK12_TDQSCK_UIFREQ2_P1_B3R0,
3009 (u4value << 3) | u4value1);
3010 /* Shuffle 2 */
3011 u4value = READ32_BITFIELD(
3012 &shu[2].rk[rnk].selph_dqsg0,
3013 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED);
3014 u4value1 = READ32_BITFIELD(
3015 &shu[2].rk[rnk].selph_dqsg1,
3016 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED);
3017 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[10],
3018 RK0_PRE_TDQSCK11_TDQSCK_UIFREQ3_B3R0,
3019 (u4value << 3) | u4value1);
3020 u4value = READ32_BITFIELD(
3021 &shu[2].rk[rnk].dqsien,
3022 SHURK0_DQSIEN_R0DQS3IEN);
3023 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[10],
3024 RK0_PRE_TDQSCK11_TDQSCK_PIFREQ3_B3R0,
3025 u4value);
3026 u4value = READ32_BITFIELD(
3027 &shu[2].rk[rnk].selph_dqsg0,
3028 SHURK0_SELPH_DQSG0_TX_DLY_DQS3_GATED_P1);
3029 u4value1 = READ32_BITFIELD(
3030 &shu[2].rk[rnk].selph_dqsg1,
3031 SHURK0_SELPH_DQSG1_REG_DLY_DQS3_GATED_P1);
3032 SET32_BITFIELDS(&rk[rnk].pre_tdqsck[11],
3033 RK0_PRE_TDQSCK12_TDQSCK_UIFREQ3_P1_B3R0,
3034 (u4value << 3) | u4value1);
3035 }
3036
3037 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[0],
3038 PRE_TDQSCK1_TDQSCK_REG_DVFS, 0x1);
3039 SET32_BITFIELDS(&ch[chn].ao.pre_tdqsck[0],
3040 PRE_TDQSCK1_TDQSCK_HW_SW_UP_SEL, 1);
3041 }
3042 }
3043
get_dram_info_after_cal(u8 * density_result,u32 rk_num)3044 void get_dram_info_after_cal(u8 *density_result, u32 rk_num)
3045 {
3046 u8 vendor_id, density, max_density = 0;
3047 u32 ddr_size, max_size = 0;
3048
3049 vendor_id = dramc_mode_reg_read_by_rank(CHANNEL_A, RANK_0, 5);
3050 dramc_show("Vendor id is %#x\n", vendor_id);
3051
3052 for (u8 rk = RANK_0; rk < rk_num; rk++) {
3053 density = dramc_mode_reg_read_by_rank(CHANNEL_A, rk, 8);
3054 dramc_dbg("MR8 %#x\n", density);
3055 density = (density >> 2) & 0xf;
3056
3057 switch (density) {
3058 case 0x0:
3059 ddr_size = 4;
3060 break;
3061 case 0x1:
3062 ddr_size = 6;
3063 break;
3064 case 0x2:
3065 ddr_size = 8;
3066 break;
3067 case 0x3:
3068 ddr_size = 12;
3069 break;
3070 case 0x4:
3071 ddr_size = 16;
3072 break;
3073 case 0x5:
3074 ddr_size = 24;
3075 break;
3076 case 0x6:
3077 ddr_size = 32;
3078 break;
3079 default:
3080 ddr_size = 0;
3081 break;
3082 }
3083 if (ddr_size > max_size) {
3084 max_size = ddr_size;
3085 max_density = density;
3086 }
3087 dramc_dbg("RK%d size %dGb, density:%d\n", rk, ddr_size, max_density);
3088 }
3089
3090 *density_result = max_density;
3091 }
3092
dramc_calibrate_all_channels(const struct sdram_params * pams,u8 freq_group,struct mr_value * mr,bool run_dvfs)3093 int dramc_calibrate_all_channels(const struct sdram_params *pams,
3094 u8 freq_group, struct mr_value *mr, bool run_dvfs)
3095 {
3096 bool fast_calib;
3097 switch (pams->source) {
3098 case DRAMC_PARAM_SOURCE_SDRAM_CONFIG:
3099 fast_calib = false;
3100 break;
3101 case DRAMC_PARAM_SOURCE_FLASH:
3102 fast_calib = true;
3103 break;
3104 default:
3105 die("Invalid DRAM param source %u\n", pams->source);
3106 return -1;
3107 }
3108
3109 bool test_passed;
3110 u8 rx_datlat[RANK_MAX] = {0};
3111 u16 osc_thrd_inc[RANK_MAX];
3112 u16 osc_thrd_dec[RANK_MAX];
3113 for (u8 chn = 0; chn < CHANNEL_MAX; chn++) {
3114 for (u8 rk = RANK_0; rk < pams->rank_num; rk++) {
3115 dramc_dbg("Start K: freq=%d, ch=%d, rank=%d\n",
3116 freq_group, chn, rk);
3117 dramc_cmd_bus_training(chn, rk, freq_group, pams,
3118 mr, run_dvfs);
3119 dramc_write_leveling(chn, rk, freq_group, pams->wr_level);
3120 dramc_auto_refresh_switch(chn, true);
3121
3122 dramc_rx_dqs_gating_cal(chn, rk, freq_group, pams,
3123 fast_calib, mr);
3124 dramc_window_perbit_cal(chn, rk, freq_group,
3125 RX_WIN_RD_DQC, pams, fast_calib);
3126 dramc_window_perbit_cal(chn, rk, freq_group,
3127 TX_WIN_DQ_DQM, pams, fast_calib);
3128 dramc_window_perbit_cal(chn, rk, freq_group,
3129 TX_WIN_DQ_ONLY, pams, fast_calib);
3130 rx_datlat[rk] = dramc_rx_datlat_cal(chn, rk, freq_group,
3131 pams, fast_calib, &test_passed);
3132 if (!test_passed)
3133 return -2;
3134 dramc_window_perbit_cal(chn, rk, freq_group,
3135 RX_WIN_TEST_ENG, pams, fast_calib);
3136 dramc_auto_refresh_switch(chn, false);
3137
3138 dqsosc_auto(chn, rk, freq_group, osc_thrd_inc, osc_thrd_dec);
3139 }
3140
3141 dqsosc_shu_settings(chn, freq_group, osc_thrd_inc, osc_thrd_dec);
3142 dramc_rx_dqs_gating_post_process(chn, freq_group, pams->rank_num);
3143 dramc_dual_rank_rx_datlat_cal(chn, freq_group, rx_datlat[0], rx_datlat[1]);
3144 }
3145 return 0;
3146 }
3147