1 /* SPDX-License-Identifier: GPL-2.0-only */
2
3 #include <assert.h>
4 #include <cbfs.h>
5 #include <cbmem.h>
6 #include <commonlib/bsd/mem_chip_info.h>
7 #include <console/console.h>
8 #include <soc/dramc_common.h>
9 #include <mrc_cache.h>
10 #include <soc/dramc_param.h>
11 #include <soc/emi.h>
12 #include <soc/mmu_operations.h>
13 #include <symbols.h>
14 #include <timer.h>
15
16 /* This must be defined in chromeos.fmd in same name and size. */
17 #define CAL_REGION_RW_MRC_CACHE "RW_MRC_CACHE"
18 #define CAL_REGION_RW_MRC_CACHE_SIZE 0x2000
19
20 _Static_assert(sizeof(struct dramc_param) <= CAL_REGION_RW_MRC_CACHE_SIZE,
21 "sizeof(struct dramc_param) exceeds " CAL_REGION_RW_MRC_CACHE);
22
23 const char *get_dram_geometry_str(u32 ddr_geometry);
24 const char *get_dram_type_str(u32 ddr_type);
25
26 static const struct ddr_base_info *curr_ddr_info;
27
mt_mem_test(const struct dramc_data * dparam)28 static int mt_mem_test(const struct dramc_data *dparam)
29 {
30 if (CONFIG(MEMORY_TEST)) {
31 u8 *addr = _dram;
32 const struct ddr_base_info *ddr_info = &dparam->ddr_info;
33
34 for (u8 rank = RANK_0; rank < ddr_info->support_ranks; rank++) {
35 int result = complex_mem_test(addr, 0x2000);
36
37 if (result != 0) {
38 printk(BIOS_ERR,
39 "[MEM] complex R/W mem test failed: %d\n", result);
40 return -1;
41 }
42 printk(BIOS_DEBUG, "[MEM] rank %u complex R/W mem test passed\n", rank);
43
44 addr += ddr_info->rank_size[rank];
45 }
46 }
47
48 return 0;
49 }
50
get_dram_geometry_str(u32 ddr_geometry)51 const char *get_dram_geometry_str(u32 ddr_geometry)
52 {
53 const char *s;
54
55 switch (ddr_geometry) {
56 case DDR_TYPE_2CH_2RK_4GB_2_2:
57 s = "2CH_2RK_4GB_2_2";
58 break;
59 case DDR_TYPE_2CH_2RK_6GB_3_3:
60 s = "2CH_2RK_6GB_3_3";
61 break;
62 case DDR_TYPE_2CH_2RK_8GB_4_4:
63 s = "2CH_2RK_8GB_4_4";
64 break;
65 case DDR_TYPE_2CH_2RK_8GB_4_4_BYTE:
66 s = "2CH_2RK_8GB_4_4_BYTE";
67 break;
68 case DDR_TYPE_2CH_1RK_4GB_4_0:
69 s = "2CH_1RK_4GB_4_0";
70 break;
71 case DDR_TYPE_2CH_2RK_6GB_2_4:
72 s = "2CH_2RK_6GB_2_4";
73 break;
74 default:
75 s = "";
76 break;
77 }
78
79 return s;
80 }
81
get_dram_type_str(u32 ddr_type)82 const char *get_dram_type_str(u32 ddr_type)
83 {
84 const char *s;
85
86 switch (ddr_type) {
87 case DDR_TYPE_DISCRETE:
88 s = "DSC";
89 break;
90 case DDR_TYPE_EMCP:
91 s = "EMCP";
92 break;
93 default:
94 s = "";
95 break;
96 }
97
98 return s;
99 }
100
mtk_dram_size(void)101 size_t mtk_dram_size(void)
102 {
103 size_t size = 0;
104
105 if (!curr_ddr_info)
106 return 0;
107 for (unsigned int i = 0; i < RANK_MAX; ++i)
108 size += curr_ddr_info->mrr_info.mr8_density[i];
109 return size;
110 }
111
fill_dram_info(struct mem_chip_info * mc,const struct ddr_base_info * ddr)112 static void fill_dram_info(struct mem_chip_info *mc, const struct ddr_base_info *ddr)
113 {
114 unsigned int c, r;
115
116 mc->num_entries = CHANNEL_MAX * ddr->mrr_info.rank_nums;
117 mc->struct_version = MEM_CHIP_STRUCT_VERSION;
118
119 struct mem_chip_entry *entry = mc->entries;
120 for (c = 0; c < CHANNEL_MAX; c++) {
121 for (r = 0; r < ddr->mrr_info.rank_nums; r++) {
122 entry->channel = c;
123 entry->rank = r;
124 entry->type = MEM_CHIP_LPDDR4X;
125 entry->channel_io_width = DQ_DATA_WIDTH_LP4;
126 entry->density_mbits = ddr->mrr_info.mr8_density[r] / CHANNEL_MAX /
127 (MiB / 8);
128 entry->io_width = DQ_DATA_WIDTH_LP4;
129 entry->manufacturer_id = ddr->mrr_info.mr5_vendor_id;
130 entry->revision_id[0] = ddr->mrr_info.mr6_revision_id;
131 entry->revision_id[1] = ddr->mrr_info.mr7_revision_id;
132 entry++;
133 }
134 }
135 }
136
add_mem_chip_info(int unused)137 static void add_mem_chip_info(int unused)
138 {
139 struct mem_chip_info *mc;
140 size_t size;
141
142 if (!CONFIG(USE_CBMEM_DRAM_INFO)) {
143 printk(BIOS_DEBUG,
144 "DRAM-K: CBMEM DRAM info is unsupported (USE_CBMEM_DRAM_INFO)\n");
145 return;
146 }
147
148 size = mem_chip_info_size(CHANNEL_MAX * curr_ddr_info->mrr_info.rank_nums);
149 mc = cbmem_add(CBMEM_ID_MEM_CHIP_INFO, size);
150 assert(mc);
151 memset(mc, 0, size);
152
153 fill_dram_info(mc, curr_ddr_info);
154 }
155 CBMEM_CREATION_HOOK(add_mem_chip_info);
156
run_dram_blob(struct dramc_param * dparam)157 static int run_dram_blob(struct dramc_param *dparam)
158 {
159 /* Load and run the provided blob for full-calibration if available */
160 struct prog dram = PROG_INIT(PROG_REFCODE, CONFIG_CBFS_PREFIX "/dram");
161
162 dump_param_header(dparam);
163
164 if (cbfs_prog_stage_load(&dram)) {
165 printk(BIOS_ERR, "DRAM-K: CBFS load program failed\n");
166 return -2;
167 }
168
169 dparam->do_putc = do_putchar;
170
171 prog_set_entry(&dram, prog_entry(&dram), dparam);
172 prog_run(&dram);
173 if (dparam->header.status != DRAMC_SUCCESS) {
174 printk(BIOS_ERR, "DRAM-K: calibration failed: status = %d\n",
175 dparam->header.status);
176 return -3;
177 }
178
179 if (!(dparam->header.config & DRAMC_CONFIG_FAST_K)
180 && !(dparam->header.flags & DRAMC_FLAG_HAS_SAVED_DATA)) {
181 printk(BIOS_ERR,
182 "DRAM-K: Full calibration executed without saving parameters. "
183 "Please ensure the blob is built properly.\n");
184 return -4;
185 }
186
187 return 0;
188 }
189
dram_run_fast_calibration(struct dramc_param * dparam)190 static int dram_run_fast_calibration(struct dramc_param *dparam)
191 {
192 const u16 config = CONFIG(MEDIATEK_DRAM_DVFS) ? DRAMC_ENABLE_DVFS : DRAMC_DISABLE_DVFS;
193
194 if (dparam->dramc_datas.ddr_info.config_dvfs != config) {
195 printk(BIOS_WARNING,
196 "DRAM-K: Incompatible config for calibration data from flash "
197 "(expected: %#x, saved: %#x)\n",
198 config, dparam->dramc_datas.ddr_info.config_dvfs);
199 return -1;
200 }
201
202 printk(BIOS_INFO, "DRAM-K: DRAM calibration data valid pass\n");
203
204 if (CONFIG(MEDIATEK_DRAM_SCRAMBLE))
205 dparam->header.config |= DRAMC_CONFIG_SCRAMBLE;
206 if (CONFIG(MEDIATEK_DRAM_BLOB_FAST_INIT)) {
207 printk(BIOS_INFO, "DRAM-K: Run fast calibration run in blob mode\n");
208
209 /*
210 * The loaded config should not contain FAST_K (done in full calibration),
211 * so we have to set that now to indicate the blob taking the config instead
212 * of generating a new config.
213 */
214 dparam->header.config |= DRAMC_CONFIG_FAST_K;
215
216 if (run_dram_blob(dparam) < 0)
217 return -3;
218 } else {
219 init_dram_by_params(dparam);
220 }
221
222 if (mt_mem_test(&dparam->dramc_datas) < 0)
223 return -4;
224
225 return 0;
226 }
227
dram_run_full_calibration(struct dramc_param * dparam)228 static int dram_run_full_calibration(struct dramc_param *dparam)
229 {
230 initialize_dramc_param(dparam);
231
232 return run_dram_blob(dparam);
233 }
234
mem_init_set_default_config(struct dramc_param * dparam,const struct sdram_info * dram_info)235 static void mem_init_set_default_config(struct dramc_param *dparam,
236 const struct sdram_info *dram_info)
237 {
238 u32 type, geometry;
239 memset(dparam, 0, sizeof(*dparam));
240
241 type = dram_info->ddr_type;
242 geometry = dram_info->ddr_geometry;
243
244 dparam->dramc_datas.ddr_info.sdram.ddr_type = type;
245
246 if (CONFIG(MEDIATEK_DRAM_DVFS))
247 dparam->dramc_datas.ddr_info.config_dvfs = DRAMC_ENABLE_DVFS;
248 if (CONFIG(MEDIATEK_DRAM_SCRAMBLE))
249 dparam->header.config |= DRAMC_CONFIG_SCRAMBLE;
250
251 dparam->dramc_datas.ddr_info.sdram.ddr_geometry = geometry;
252
253 printk(BIOS_INFO, "DRAM-K: ddr_type: %s, config_dvfs: %d, ddr_geometry: %s\n",
254 get_dram_type_str(type),
255 dparam->dramc_datas.ddr_info.config_dvfs,
256 get_dram_geometry_str(geometry));
257 }
258
mt_mem_init_run(struct dramc_param * dparam,const struct sdram_info * dram_info)259 static void mt_mem_init_run(struct dramc_param *dparam,
260 const struct sdram_info *dram_info)
261 {
262 const ssize_t mrc_cache_size = sizeof(*dparam);
263 ssize_t data_size;
264 struct stopwatch sw;
265 int ret;
266
267 /* Load calibration params from flash and run fast calibration */
268 data_size = mrc_cache_load_current(MRC_TRAINING_DATA,
269 DRAMC_PARAM_HEADER_VERSION,
270 dparam, mrc_cache_size);
271 if (data_size == mrc_cache_size) {
272 printk(BIOS_INFO, "DRAM-K: Running fast calibration\n");
273 stopwatch_init(&sw);
274
275 ret = dram_run_fast_calibration(dparam);
276 if (ret != 0) {
277 printk(BIOS_ERR, "DRAM-K: Failed to run fast calibration "
278 "in %lld msecs, error: %d\n",
279 stopwatch_duration_msecs(&sw), ret);
280
281 /* Erase flash data after fast calibration failed */
282 memset(dparam, 0xa5, mrc_cache_size);
283 mrc_cache_stash_data(MRC_TRAINING_DATA,
284 DRAMC_PARAM_HEADER_VERSION,
285 dparam, mrc_cache_size);
286 } else {
287 printk(BIOS_INFO, "DRAM-K: Fast calibration passed in %lld msecs\n",
288 stopwatch_duration_msecs(&sw));
289 return;
290 }
291 } else {
292 printk(BIOS_WARNING, "DRAM-K: Invalid data in flash (size: %#zx, expected: %#zx)\n",
293 data_size, mrc_cache_size);
294 }
295
296 /* Run full calibration */
297 printk(BIOS_INFO, "DRAM-K: Running full calibration\n");
298 mem_init_set_default_config(dparam, dram_info);
299
300 stopwatch_init(&sw);
301 int err = dram_run_full_calibration(dparam);
302 if (err == 0) {
303 printk(BIOS_INFO, "DRAM-K: Full calibration passed in %lld msecs\n",
304 stopwatch_duration_msecs(&sw));
305 mrc_cache_stash_data(MRC_TRAINING_DATA,
306 DRAMC_PARAM_HEADER_VERSION,
307 dparam, mrc_cache_size);
308 } else {
309 printk(BIOS_ERR, "DRAM-K: Full calibration failed in %lld msecs\n",
310 stopwatch_duration_msecs(&sw));
311 }
312 }
313
mt_mem_init(struct dramc_param * dparam)314 void mt_mem_init(struct dramc_param *dparam)
315 {
316 const struct sdram_info *sdram_param = NULL;
317 static struct sdram_info fake_sdram_param;
318
319 if (CONFIG(MEDIATEK_DRAM_ADAPTIVE))
320 sdram_param = &fake_sdram_param;
321 else
322 sdram_param = get_sdram_config();
323
324 mt_mem_init_run(dparam, sdram_param);
325 }
326
mtk_dram_init(void)327 void mtk_dram_init(void)
328 {
329 /* dramc_param is too large to fit in stack. */
330 static struct dramc_param dramc_parameter;
331 mt_mem_init(&dramc_parameter);
332 curr_ddr_info = &dramc_parameter.dramc_datas.ddr_info;
333 mtk_mmu_after_dram();
334 }
335