1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * UFS Host Controller driver for Exynos specific extensions
4  *
5  * Copyright (C) 2014-2015 Samsung Electronics Co., Ltd.
6  * Author: Seungwon Jeon  <[email protected]>
7  * Author: Alim Akhtar <[email protected]>
8  *
9  */
10 
11 #include <linux/unaligned.h>
12 #include <crypto/aes.h>
13 #include <linux/arm-smccc.h>
14 #include <linux/clk.h>
15 #include <linux/delay.h>
16 #include <linux/module.h>
17 #include <linux/of.h>
18 #include <linux/of_address.h>
19 #include <linux/mfd/syscon.h>
20 #include <linux/phy/phy.h>
21 #include <linux/platform_device.h>
22 #include <linux/regmap.h>
23 
24 #include <ufs/ufshcd.h>
25 #include "ufshcd-pltfrm.h"
26 #include <ufs/ufshci.h>
27 #include <ufs/unipro.h>
28 
29 #include "ufs-exynos.h"
30 
31 #define DATA_UNIT_SIZE		4096
32 
33 /*
34  * Exynos's Vendor specific registers for UFSHCI
35  */
36 #define HCI_TXPRDT_ENTRY_SIZE	0x00
37 #define PRDT_PREFECT_EN		BIT(31)
38 #define HCI_RXPRDT_ENTRY_SIZE	0x04
39 #define HCI_1US_TO_CNT_VAL	0x0C
40 #define CNT_VAL_1US_MASK	0x3FF
41 #define HCI_UTRL_NEXUS_TYPE	0x40
42 #define HCI_UTMRL_NEXUS_TYPE	0x44
43 #define HCI_SW_RST		0x50
44 #define UFS_LINK_SW_RST		BIT(0)
45 #define UFS_UNIPRO_SW_RST	BIT(1)
46 #define UFS_SW_RST_MASK		(UFS_UNIPRO_SW_RST | UFS_LINK_SW_RST)
47 #define HCI_DATA_REORDER	0x60
48 #define HCI_UNIPRO_APB_CLK_CTRL	0x68
49 #define UNIPRO_APB_CLK(v, x)	(((v) & ~0xF) | ((x) & 0xF))
50 #define HCI_AXIDMA_RWDATA_BURST_LEN	0x6C
51 #define WLU_EN			BIT(31)
52 #define WLU_BURST_LEN(x)	((x) << 27 | ((x) & 0xF))
53 #define HCI_GPIO_OUT		0x70
54 #define HCI_ERR_EN_PA_LAYER	0x78
55 #define HCI_ERR_EN_DL_LAYER	0x7C
56 #define HCI_ERR_EN_N_LAYER	0x80
57 #define HCI_ERR_EN_T_LAYER	0x84
58 #define HCI_ERR_EN_DME_LAYER	0x88
59 #define HCI_V2P1_CTRL		0x8C
60 #define IA_TICK_SEL		BIT(16)
61 #define HCI_CLKSTOP_CTRL	0xB0
62 #define REFCLKOUT_STOP		BIT(4)
63 #define MPHY_APBCLK_STOP	BIT(3)
64 #define REFCLK_STOP		BIT(2)
65 #define UNIPRO_MCLK_STOP	BIT(1)
66 #define UNIPRO_PCLK_STOP	BIT(0)
67 #define CLK_STOP_MASK		(REFCLKOUT_STOP | REFCLK_STOP |\
68 				 UNIPRO_MCLK_STOP | MPHY_APBCLK_STOP|\
69 				 UNIPRO_PCLK_STOP)
70 /* HCI_MISC is also known as HCI_FORCE_HCS */
71 #define HCI_MISC		0xB4
72 #define REFCLK_CTRL_EN		BIT(7)
73 #define UNIPRO_PCLK_CTRL_EN	BIT(6)
74 #define UNIPRO_MCLK_CTRL_EN	BIT(5)
75 #define HCI_CORECLK_CTRL_EN	BIT(4)
76 #define CLK_CTRL_EN_MASK	(REFCLK_CTRL_EN |\
77 				 UNIPRO_PCLK_CTRL_EN |\
78 				 UNIPRO_MCLK_CTRL_EN)
79 
80 #define HCI_IOP_ACG_DISABLE	0x100
81 #define HCI_IOP_ACG_DISABLE_EN	BIT(0)
82 
83 /* Device fatal error */
84 #define DFES_ERR_EN		BIT(31)
85 #define DFES_DEF_L2_ERRS	(UIC_DATA_LINK_LAYER_ERROR_RX_BUF_OF |\
86 				 UIC_DATA_LINK_LAYER_ERROR_PA_INIT)
87 #define DFES_DEF_L3_ERRS	(UIC_NETWORK_UNSUPPORTED_HEADER_TYPE |\
88 				 UIC_NETWORK_BAD_DEVICEID_ENC |\
89 				 UIC_NETWORK_LHDR_TRAP_PACKET_DROPPING)
90 #define DFES_DEF_L4_ERRS	(UIC_TRANSPORT_UNSUPPORTED_HEADER_TYPE |\
91 				 UIC_TRANSPORT_UNKNOWN_CPORTID |\
92 				 UIC_TRANSPORT_NO_CONNECTION_RX |\
93 				 UIC_TRANSPORT_BAD_TC)
94 
95 /* UFS Shareability */
96 #define UFS_EXYNOSAUTO_WR_SHARABLE	BIT(2)
97 #define UFS_EXYNOSAUTO_RD_SHARABLE	BIT(1)
98 #define UFS_EXYNOSAUTO_SHARABLE		(UFS_EXYNOSAUTO_WR_SHARABLE | \
99 					 UFS_EXYNOSAUTO_RD_SHARABLE)
100 #define UFS_GS101_WR_SHARABLE		BIT(1)
101 #define UFS_GS101_RD_SHARABLE		BIT(0)
102 #define UFS_GS101_SHARABLE		(UFS_GS101_WR_SHARABLE | \
103 					 UFS_GS101_RD_SHARABLE)
104 #define UFS_SHAREABILITY_OFFSET		0x710
105 
106 /* Multi-host registers */
107 #define MHCTRL			0xC4
108 #define MHCTRL_EN_VH_MASK	(0xE)
109 #define MHCTRL_EN_VH(vh)	(vh << 1)
110 #define PH2VH_MBOX		0xD8
111 
112 #define MH_MSG_MASK		(0xFF)
113 
114 #define MH_MSG(id, msg)		((id << 8) | (msg & 0xFF))
115 #define MH_MSG_PH_READY		0x1
116 #define MH_MSG_VH_READY		0x2
117 
118 #define ALLOW_INQUIRY		BIT(25)
119 #define ALLOW_MODE_SELECT	BIT(24)
120 #define ALLOW_MODE_SENSE	BIT(23)
121 #define ALLOW_PRE_FETCH		GENMASK(22, 21)
122 #define ALLOW_READ_CMD_ALL	GENMASK(20, 18)	/* read_6/10/16 */
123 #define ALLOW_READ_BUFFER	BIT(17)
124 #define ALLOW_READ_CAPACITY	GENMASK(16, 15)
125 #define ALLOW_REPORT_LUNS	BIT(14)
126 #define ALLOW_REQUEST_SENSE	BIT(13)
127 #define ALLOW_SYNCHRONIZE_CACHE	GENMASK(8, 7)
128 #define ALLOW_TEST_UNIT_READY	BIT(6)
129 #define ALLOW_UNMAP		BIT(5)
130 #define ALLOW_VERIFY		BIT(4)
131 #define ALLOW_WRITE_CMD_ALL	GENMASK(3, 1)	/* write_6/10/16 */
132 
133 #define ALLOW_TRANS_VH_DEFAULT	(ALLOW_INQUIRY | ALLOW_MODE_SELECT | \
134 				 ALLOW_MODE_SENSE | ALLOW_PRE_FETCH | \
135 				 ALLOW_READ_CMD_ALL | ALLOW_READ_BUFFER | \
136 				 ALLOW_READ_CAPACITY | ALLOW_REPORT_LUNS | \
137 				 ALLOW_REQUEST_SENSE | ALLOW_SYNCHRONIZE_CACHE | \
138 				 ALLOW_TEST_UNIT_READY | ALLOW_UNMAP | \
139 				 ALLOW_VERIFY | ALLOW_WRITE_CMD_ALL)
140 
141 #define HCI_MH_ALLOWABLE_TRAN_OF_VH		0x30C
142 #define HCI_MH_IID_IN_TASK_TAG			0X308
143 
144 #define PH_READY_TIMEOUT_MS			(5 * MSEC_PER_SEC)
145 
146 enum {
147 	UNIPRO_L1_5 = 0,/* PHY Adapter */
148 	UNIPRO_L2,	/* Data Link */
149 	UNIPRO_L3,	/* Network */
150 	UNIPRO_L4,	/* Transport */
151 	UNIPRO_DME,	/* DME */
152 };
153 
154 /*
155  * UNIPRO registers
156  */
157 #define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0	0x7888
158 #define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1	0x788c
159 #define UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2	0x7890
160 #define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0	0x78B8
161 #define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1	0x78BC
162 #define UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2	0x78C0
163 
164 /*
165  * UFS Protector registers
166  */
167 #define UFSPRSECURITY	0x010
168 #define NSSMU		BIT(14)
169 #define UFSPSBEGIN0	0x200
170 #define UFSPSEND0	0x204
171 #define UFSPSLUN0	0x208
172 #define UFSPSCTRL0	0x20C
173 
174 #define CNTR_DIV_VAL 40
175 
176 static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en);
177 static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en);
178 
exynos_ufs_enable_auto_ctrl_hcc(struct exynos_ufs * ufs)179 static inline void exynos_ufs_enable_auto_ctrl_hcc(struct exynos_ufs *ufs)
180 {
181 	exynos_ufs_auto_ctrl_hcc(ufs, true);
182 }
183 
exynos_ufs_disable_auto_ctrl_hcc(struct exynos_ufs * ufs)184 static inline void exynos_ufs_disable_auto_ctrl_hcc(struct exynos_ufs *ufs)
185 {
186 	exynos_ufs_auto_ctrl_hcc(ufs, false);
187 }
188 
exynos_ufs_disable_auto_ctrl_hcc_save(struct exynos_ufs * ufs,u32 * val)189 static inline void exynos_ufs_disable_auto_ctrl_hcc_save(
190 					struct exynos_ufs *ufs, u32 *val)
191 {
192 	*val = hci_readl(ufs, HCI_MISC);
193 	exynos_ufs_auto_ctrl_hcc(ufs, false);
194 }
195 
exynos_ufs_auto_ctrl_hcc_restore(struct exynos_ufs * ufs,u32 * val)196 static inline void exynos_ufs_auto_ctrl_hcc_restore(
197 					struct exynos_ufs *ufs, u32 *val)
198 {
199 	hci_writel(ufs, *val, HCI_MISC);
200 }
201 
exynos_ufs_gate_clks(struct exynos_ufs * ufs)202 static inline void exynos_ufs_gate_clks(struct exynos_ufs *ufs)
203 {
204 	exynos_ufs_ctrl_clkstop(ufs, true);
205 }
206 
exynos_ufs_ungate_clks(struct exynos_ufs * ufs)207 static inline void exynos_ufs_ungate_clks(struct exynos_ufs *ufs)
208 {
209 	exynos_ufs_ctrl_clkstop(ufs, false);
210 }
211 
exynos_ufs_shareability(struct exynos_ufs * ufs)212 static int exynos_ufs_shareability(struct exynos_ufs *ufs)
213 {
214 	/* IO Coherency setting */
215 	if (ufs->sysreg) {
216 		return regmap_update_bits(ufs->sysreg,
217 					  ufs->iocc_offset,
218 					  ufs->iocc_mask, ufs->iocc_val);
219 	}
220 
221 	return 0;
222 }
223 
gs101_ufs_drv_init(struct exynos_ufs * ufs)224 static int gs101_ufs_drv_init(struct exynos_ufs *ufs)
225 {
226 	struct ufs_hba *hba = ufs->hba;
227 	u32 reg;
228 
229 	/* Enable WriteBooster */
230 	hba->caps |= UFSHCD_CAP_WB_EN;
231 
232 	/* Enable clock gating and hibern8 */
233 	hba->caps |= UFSHCD_CAP_CLK_GATING | UFSHCD_CAP_HIBERN8_WITH_CLK_GATING;
234 
235 	/* set ACG to be controlled by UFS_ACG_DISABLE */
236 	reg = hci_readl(ufs, HCI_IOP_ACG_DISABLE);
237 	hci_writel(ufs, reg & (~HCI_IOP_ACG_DISABLE_EN), HCI_IOP_ACG_DISABLE);
238 
239 	return exynos_ufs_shareability(ufs);
240 }
241 
exynosauto_ufs_drv_init(struct exynos_ufs * ufs)242 static int exynosauto_ufs_drv_init(struct exynos_ufs *ufs)
243 {
244 	return exynos_ufs_shareability(ufs);
245 }
246 
exynosauto_ufs_post_hce_enable(struct exynos_ufs * ufs)247 static int exynosauto_ufs_post_hce_enable(struct exynos_ufs *ufs)
248 {
249 	struct ufs_hba *hba = ufs->hba;
250 
251 	/* Enable Virtual Host #1 */
252 	ufshcd_rmwl(hba, MHCTRL_EN_VH_MASK, MHCTRL_EN_VH(1), MHCTRL);
253 	/* Default VH Transfer permissions */
254 	hci_writel(ufs, ALLOW_TRANS_VH_DEFAULT, HCI_MH_ALLOWABLE_TRAN_OF_VH);
255 	/* IID information is replaced in TASKTAG[7:5] instead of IID in UCD */
256 	hci_writel(ufs, 0x1, HCI_MH_IID_IN_TASK_TAG);
257 
258 	return 0;
259 }
260 
exynosauto_ufs_pre_link(struct exynos_ufs * ufs)261 static int exynosauto_ufs_pre_link(struct exynos_ufs *ufs)
262 {
263 	struct ufs_hba *hba = ufs->hba;
264 	int i;
265 	u32 tx_line_reset_period, rx_line_reset_period;
266 
267 	rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate) / NSEC_PER_MSEC;
268 	tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate) / NSEC_PER_MSEC;
269 
270 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40);
271 	for_each_ufs_rx_lane(ufs, i) {
272 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i),
273 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
274 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0);
275 
276 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i),
277 			       (rx_line_reset_period >> 16) & 0xFF);
278 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i),
279 			       (rx_line_reset_period >> 8) & 0xFF);
280 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i),
281 			       (rx_line_reset_period) & 0xFF);
282 
283 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x79);
284 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1);
285 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6);
286 	}
287 
288 	for_each_ufs_tx_lane(ufs, i) {
289 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i),
290 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
291 		/* Not to affect VND_TX_LINERESET_PVALUE to VND_TX_CLK_PRD */
292 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i),
293 			       0x02);
294 
295 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i),
296 			       (tx_line_reset_period >> 16) & 0xFF);
297 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i),
298 			       (tx_line_reset_period >> 8) & 0xFF);
299 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i),
300 			       (tx_line_reset_period) & 0xFF);
301 
302 		/* TX PWM Gear Capability / PWM_G1_ONLY */
303 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 0x1);
304 	}
305 
306 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0);
307 
308 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0);
309 
310 	ufshcd_dme_set(hba, UIC_ARG_MIB(0xa011), 0x8000);
311 
312 	return 0;
313 }
314 
exynosauto_ufs_pre_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)315 static int exynosauto_ufs_pre_pwr_change(struct exynos_ufs *ufs,
316 					 struct ufs_pa_layer_attr *pwr)
317 {
318 	struct ufs_hba *hba = ufs->hba;
319 
320 	/* PACP_PWR_req and delivered to the remote DME */
321 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000);
322 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000);
323 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000);
324 
325 	return 0;
326 }
327 
exynosauto_ufs_post_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)328 static int exynosauto_ufs_post_pwr_change(struct exynos_ufs *ufs,
329 					  struct ufs_pa_layer_attr *pwr)
330 {
331 	struct ufs_hba *hba = ufs->hba;
332 	u32 enabled_vh;
333 
334 	enabled_vh = ufshcd_readl(hba, MHCTRL) & MHCTRL_EN_VH_MASK;
335 
336 	/* Send physical host ready message to virtual hosts */
337 	ufshcd_writel(hba, MH_MSG(enabled_vh, MH_MSG_PH_READY), PH2VH_MBOX);
338 
339 	return 0;
340 }
341 
exynos7_ufs_pre_link(struct exynos_ufs * ufs)342 static int exynos7_ufs_pre_link(struct exynos_ufs *ufs)
343 {
344 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
345 	u32 val = attr->pa_dbg_opt_suite1_val;
346 	struct ufs_hba *hba = ufs->hba;
347 	int i;
348 
349 	exynos_ufs_enable_ov_tm(hba);
350 	for_each_ufs_tx_lane(ufs, i)
351 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x297, i), 0x17);
352 	for_each_ufs_rx_lane(ufs, i) {
353 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x362, i), 0xff);
354 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x363, i), 0x00);
355 	}
356 	exynos_ufs_disable_ov_tm(hba);
357 
358 	for_each_ufs_tx_lane(ufs, i)
359 		ufshcd_dme_set(hba,
360 			UIC_ARG_MIB_SEL(TX_HIBERN8_CONTROL, i), 0x0);
361 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_TXPHY_CFGUPDT), 0x1);
362 	udelay(1);
363 	ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off),
364 					val | (1 << 12));
365 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_RESET_PHY), 0x1);
366 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_SKIP_LINE_RESET), 0x1);
367 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_LINE_RESET_REQ), 0x1);
368 	udelay(1600);
369 	ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off), val);
370 
371 	return 0;
372 }
373 
exynos7_ufs_post_link(struct exynos_ufs * ufs)374 static int exynos7_ufs_post_link(struct exynos_ufs *ufs)
375 {
376 	struct ufs_hba *hba = ufs->hba;
377 	int i;
378 
379 	exynos_ufs_enable_ov_tm(hba);
380 	for_each_ufs_tx_lane(ufs, i) {
381 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x28b, i), 0x83);
382 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x29a, i), 0x07);
383 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x277, i),
384 			TX_LINERESET_N(exynos_ufs_calc_time_cntr(ufs, 200000)));
385 	}
386 	exynos_ufs_disable_ov_tm(hba);
387 
388 	exynos_ufs_enable_dbg_mode(hba);
389 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0xbb8);
390 	exynos_ufs_disable_dbg_mode(hba);
391 
392 	return 0;
393 }
394 
exynos7_ufs_pre_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)395 static int exynos7_ufs_pre_pwr_change(struct exynos_ufs *ufs,
396 						struct ufs_pa_layer_attr *pwr)
397 {
398 	unipro_writel(ufs, 0x22, UNIPRO_DBG_FORCE_DME_CTRL_STATE);
399 
400 	return 0;
401 }
402 
exynos7_ufs_post_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)403 static int exynos7_ufs_post_pwr_change(struct exynos_ufs *ufs,
404 						struct ufs_pa_layer_attr *pwr)
405 {
406 	struct ufs_hba *hba = ufs->hba;
407 	int lanes = max_t(u32, pwr->lane_rx, pwr->lane_tx);
408 
409 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_RXPHY_CFGUPDT), 0x1);
410 
411 	if (lanes == 1) {
412 		exynos_ufs_enable_dbg_mode(hba);
413 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_CONNECTEDTXDATALANES), 0x1);
414 		exynos_ufs_disable_dbg_mode(hba);
415 	}
416 
417 	return 0;
418 }
419 
420 /*
421  * exynos_ufs_auto_ctrl_hcc - HCI core clock control by h/w
422  * Control should be disabled in the below cases
423  * - Before host controller S/W reset
424  * - Access to UFS protector's register
425  */
exynos_ufs_auto_ctrl_hcc(struct exynos_ufs * ufs,bool en)426 static void exynos_ufs_auto_ctrl_hcc(struct exynos_ufs *ufs, bool en)
427 {
428 	u32 misc = hci_readl(ufs, HCI_MISC);
429 
430 	if (en)
431 		hci_writel(ufs, misc | HCI_CORECLK_CTRL_EN, HCI_MISC);
432 	else
433 		hci_writel(ufs, misc & ~HCI_CORECLK_CTRL_EN, HCI_MISC);
434 }
435 
exynos_ufs_ctrl_clkstop(struct exynos_ufs * ufs,bool en)436 static void exynos_ufs_ctrl_clkstop(struct exynos_ufs *ufs, bool en)
437 {
438 	u32 ctrl = hci_readl(ufs, HCI_CLKSTOP_CTRL);
439 	u32 misc = hci_readl(ufs, HCI_MISC);
440 
441 	if (en) {
442 		hci_writel(ufs, misc | CLK_CTRL_EN_MASK, HCI_MISC);
443 		hci_writel(ufs, ctrl | CLK_STOP_MASK, HCI_CLKSTOP_CTRL);
444 	} else {
445 		hci_writel(ufs, ctrl & ~CLK_STOP_MASK, HCI_CLKSTOP_CTRL);
446 		hci_writel(ufs, misc & ~CLK_CTRL_EN_MASK, HCI_MISC);
447 	}
448 }
449 
exynos_ufs_get_clk_info(struct exynos_ufs * ufs)450 static int exynos_ufs_get_clk_info(struct exynos_ufs *ufs)
451 {
452 	struct ufs_hba *hba = ufs->hba;
453 	struct list_head *head = &hba->clk_list_head;
454 	struct ufs_clk_info *clki;
455 	unsigned long pclk_rate;
456 	u32 f_min, f_max;
457 	u8 div = 0;
458 	int ret = 0;
459 
460 	if (list_empty(head))
461 		goto out;
462 
463 	list_for_each_entry(clki, head, list) {
464 		if (!IS_ERR(clki->clk)) {
465 			if (!strcmp(clki->name, "core_clk"))
466 				ufs->clk_hci_core = clki->clk;
467 			else if (!strcmp(clki->name, "sclk_unipro_main"))
468 				ufs->clk_unipro_main = clki->clk;
469 		}
470 	}
471 
472 	if (!ufs->clk_hci_core || !ufs->clk_unipro_main) {
473 		dev_err(hba->dev, "failed to get clk info\n");
474 		ret = -EINVAL;
475 		goto out;
476 	}
477 
478 	ufs->mclk_rate = clk_get_rate(ufs->clk_unipro_main);
479 	pclk_rate = clk_get_rate(ufs->clk_hci_core);
480 	f_min = ufs->pclk_avail_min;
481 	f_max = ufs->pclk_avail_max;
482 
483 	if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) {
484 		do {
485 			pclk_rate /= (div + 1);
486 
487 			if (pclk_rate <= f_max)
488 				break;
489 			div++;
490 		} while (pclk_rate >= f_min);
491 	}
492 
493 	if (unlikely(pclk_rate < f_min || pclk_rate > f_max)) {
494 		dev_err(hba->dev, "not available pclk range %lu\n", pclk_rate);
495 		ret = -EINVAL;
496 		goto out;
497 	}
498 
499 	ufs->pclk_rate = pclk_rate;
500 	ufs->pclk_div = div;
501 
502 out:
503 	return ret;
504 }
505 
exynos_ufs_set_unipro_pclk_div(struct exynos_ufs * ufs)506 static void exynos_ufs_set_unipro_pclk_div(struct exynos_ufs *ufs)
507 {
508 	if (ufs->opts & EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL) {
509 		u32 val;
510 
511 		val = hci_readl(ufs, HCI_UNIPRO_APB_CLK_CTRL);
512 		hci_writel(ufs, UNIPRO_APB_CLK(val, ufs->pclk_div),
513 			   HCI_UNIPRO_APB_CLK_CTRL);
514 	}
515 }
516 
exynos_ufs_set_pwm_clk_div(struct exynos_ufs * ufs)517 static void exynos_ufs_set_pwm_clk_div(struct exynos_ufs *ufs)
518 {
519 	struct ufs_hba *hba = ufs->hba;
520 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
521 
522 	ufshcd_dme_set(hba,
523 		UIC_ARG_MIB(CMN_PWM_CLK_CTRL), attr->cmn_pwm_clk_ctrl);
524 }
525 
exynos_ufs_calc_pwm_clk_div(struct exynos_ufs * ufs)526 static void exynos_ufs_calc_pwm_clk_div(struct exynos_ufs *ufs)
527 {
528 	struct ufs_hba *hba = ufs->hba;
529 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
530 	const unsigned int div = 30, mult = 20;
531 	const unsigned long pwm_min = 3 * 1000 * 1000;
532 	const unsigned long pwm_max = 9 * 1000 * 1000;
533 	const int divs[] = {32, 16, 8, 4};
534 	unsigned long clk = 0, _clk, clk_period;
535 	int i = 0, clk_idx = -1;
536 
537 	clk_period = UNIPRO_PCLK_PERIOD(ufs);
538 	for (i = 0; i < ARRAY_SIZE(divs); i++) {
539 		_clk = NSEC_PER_SEC * mult / (clk_period * divs[i] * div);
540 		if (_clk >= pwm_min && _clk <= pwm_max) {
541 			if (_clk > clk) {
542 				clk_idx = i;
543 				clk = _clk;
544 			}
545 		}
546 	}
547 
548 	if (clk_idx == -1) {
549 		ufshcd_dme_get(hba, UIC_ARG_MIB(CMN_PWM_CLK_CTRL), &clk_idx);
550 		dev_err(hba->dev,
551 			"failed to decide pwm clock divider, will not change\n");
552 	}
553 
554 	attr->cmn_pwm_clk_ctrl = clk_idx & PWM_CLK_CTRL_MASK;
555 }
556 
exynos_ufs_calc_time_cntr(struct exynos_ufs * ufs,long period)557 long exynos_ufs_calc_time_cntr(struct exynos_ufs *ufs, long period)
558 {
559 	const int precise = 10;
560 	long pclk_rate = ufs->pclk_rate;
561 	long clk_period, fraction;
562 
563 	clk_period = UNIPRO_PCLK_PERIOD(ufs);
564 	fraction = ((NSEC_PER_SEC % pclk_rate) * precise) / pclk_rate;
565 
566 	return (period * precise) / ((clk_period * precise) + fraction);
567 }
568 
exynos_ufs_specify_phy_time_attr(struct exynos_ufs * ufs)569 static void exynos_ufs_specify_phy_time_attr(struct exynos_ufs *ufs)
570 {
571 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
572 	struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg;
573 
574 	if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR)
575 		return;
576 
577 	t_cfg->tx_linereset_p =
578 		exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_p_nsec);
579 	t_cfg->tx_linereset_n =
580 		exynos_ufs_calc_time_cntr(ufs, attr->tx_dif_n_nsec);
581 	t_cfg->tx_high_z_cnt =
582 		exynos_ufs_calc_time_cntr(ufs, attr->tx_high_z_cnt_nsec);
583 	t_cfg->tx_base_n_val =
584 		exynos_ufs_calc_time_cntr(ufs, attr->tx_base_unit_nsec);
585 	t_cfg->tx_gran_n_val =
586 		exynos_ufs_calc_time_cntr(ufs, attr->tx_gran_unit_nsec);
587 	t_cfg->tx_sleep_cnt =
588 		exynos_ufs_calc_time_cntr(ufs, attr->tx_sleep_cnt);
589 
590 	t_cfg->rx_linereset =
591 		exynos_ufs_calc_time_cntr(ufs, attr->rx_dif_p_nsec);
592 	t_cfg->rx_hibern8_wait =
593 		exynos_ufs_calc_time_cntr(ufs, attr->rx_hibern8_wait_nsec);
594 	t_cfg->rx_base_n_val =
595 		exynos_ufs_calc_time_cntr(ufs, attr->rx_base_unit_nsec);
596 	t_cfg->rx_gran_n_val =
597 		exynos_ufs_calc_time_cntr(ufs, attr->rx_gran_unit_nsec);
598 	t_cfg->rx_sleep_cnt =
599 		exynos_ufs_calc_time_cntr(ufs, attr->rx_sleep_cnt);
600 	t_cfg->rx_stall_cnt =
601 		exynos_ufs_calc_time_cntr(ufs, attr->rx_stall_cnt);
602 }
603 
exynos_ufs_config_phy_time_attr(struct exynos_ufs * ufs)604 static void exynos_ufs_config_phy_time_attr(struct exynos_ufs *ufs)
605 {
606 	struct ufs_hba *hba = ufs->hba;
607 	struct ufs_phy_time_cfg *t_cfg = &ufs->t_cfg;
608 	int i;
609 
610 	exynos_ufs_set_pwm_clk_div(ufs);
611 
612 	exynos_ufs_enable_ov_tm(hba);
613 
614 	for_each_ufs_rx_lane(ufs, i) {
615 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_FILLER_ENABLE, i),
616 				ufs->drv_data->uic_attr->rx_filler_enable);
617 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_LINERESET_VAL, i),
618 				RX_LINERESET(t_cfg->rx_linereset));
619 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_07_00, i),
620 				RX_BASE_NVAL_L(t_cfg->rx_base_n_val));
621 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_BASE_NVAL_15_08, i),
622 				RX_BASE_NVAL_H(t_cfg->rx_base_n_val));
623 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_07_00, i),
624 				RX_GRAN_NVAL_L(t_cfg->rx_gran_n_val));
625 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_GRAN_NVAL_10_08, i),
626 				RX_GRAN_NVAL_H(t_cfg->rx_gran_n_val));
627 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_SLEEP_CNT_TIMER, i),
628 				RX_OV_SLEEP_CNT(t_cfg->rx_sleep_cnt));
629 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(RX_OV_STALL_CNT_TIMER, i),
630 				RX_OV_STALL_CNT(t_cfg->rx_stall_cnt));
631 	}
632 
633 	for_each_ufs_tx_lane(ufs, i) {
634 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_LINERESET_P_VAL, i),
635 				TX_LINERESET_P(t_cfg->tx_linereset_p));
636 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_07_00, i),
637 				TX_HIGH_Z_CNT_L(t_cfg->tx_high_z_cnt));
638 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_HIGH_Z_CNT_11_08, i),
639 				TX_HIGH_Z_CNT_H(t_cfg->tx_high_z_cnt));
640 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_07_00, i),
641 				TX_BASE_NVAL_L(t_cfg->tx_base_n_val));
642 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_BASE_NVAL_15_08, i),
643 				TX_BASE_NVAL_H(t_cfg->tx_base_n_val));
644 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_07_00, i),
645 				TX_GRAN_NVAL_L(t_cfg->tx_gran_n_val));
646 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_GRAN_NVAL_10_08, i),
647 				TX_GRAN_NVAL_H(t_cfg->tx_gran_n_val));
648 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_OV_SLEEP_CNT_TIMER, i),
649 				TX_OV_H8_ENTER_EN |
650 				TX_OV_SLEEP_CNT(t_cfg->tx_sleep_cnt));
651 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(TX_MIN_ACTIVATETIME, i),
652 				ufs->drv_data->uic_attr->tx_min_activatetime);
653 	}
654 
655 	exynos_ufs_disable_ov_tm(hba);
656 }
657 
exynos_ufs_config_phy_cap_attr(struct exynos_ufs * ufs)658 static void exynos_ufs_config_phy_cap_attr(struct exynos_ufs *ufs)
659 {
660 	struct ufs_hba *hba = ufs->hba;
661 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
662 	int i;
663 
664 	exynos_ufs_enable_ov_tm(hba);
665 
666 	for_each_ufs_rx_lane(ufs, i) {
667 		ufshcd_dme_set(hba,
668 				UIC_ARG_MIB_SEL(RX_HS_G1_SYNC_LENGTH_CAP, i),
669 				attr->rx_hs_g1_sync_len_cap);
670 		ufshcd_dme_set(hba,
671 				UIC_ARG_MIB_SEL(RX_HS_G2_SYNC_LENGTH_CAP, i),
672 				attr->rx_hs_g2_sync_len_cap);
673 		ufshcd_dme_set(hba,
674 				UIC_ARG_MIB_SEL(RX_HS_G3_SYNC_LENGTH_CAP, i),
675 				attr->rx_hs_g3_sync_len_cap);
676 		ufshcd_dme_set(hba,
677 				UIC_ARG_MIB_SEL(RX_HS_G1_PREP_LENGTH_CAP, i),
678 				attr->rx_hs_g1_prep_sync_len_cap);
679 		ufshcd_dme_set(hba,
680 				UIC_ARG_MIB_SEL(RX_HS_G2_PREP_LENGTH_CAP, i),
681 				attr->rx_hs_g2_prep_sync_len_cap);
682 		ufshcd_dme_set(hba,
683 				UIC_ARG_MIB_SEL(RX_HS_G3_PREP_LENGTH_CAP, i),
684 				attr->rx_hs_g3_prep_sync_len_cap);
685 	}
686 
687 	if (attr->rx_adv_fine_gran_sup_en == 0) {
688 		for_each_ufs_rx_lane(ufs, i) {
689 			ufshcd_dme_set(hba,
690 				UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP, i), 0);
691 
692 			if (attr->rx_min_actv_time_cap)
693 				ufshcd_dme_set(hba,
694 					UIC_ARG_MIB_SEL(
695 					RX_MIN_ACTIVATETIME_CAPABILITY, i),
696 					attr->rx_min_actv_time_cap);
697 
698 			if (attr->rx_hibern8_time_cap)
699 				ufshcd_dme_set(hba,
700 					UIC_ARG_MIB_SEL(RX_HIBERN8TIME_CAP, i),
701 						attr->rx_hibern8_time_cap);
702 		}
703 	} else if (attr->rx_adv_fine_gran_sup_en == 1) {
704 		for_each_ufs_rx_lane(ufs, i) {
705 			if (attr->rx_adv_fine_gran_step)
706 				ufshcd_dme_set(hba,
707 					UIC_ARG_MIB_SEL(RX_ADV_GRANULARITY_CAP,
708 						i), RX_ADV_FINE_GRAN_STEP(
709 						attr->rx_adv_fine_gran_step));
710 
711 			if (attr->rx_adv_min_actv_time_cap)
712 				ufshcd_dme_set(hba,
713 					UIC_ARG_MIB_SEL(
714 						RX_ADV_MIN_ACTIVATETIME_CAP, i),
715 						attr->rx_adv_min_actv_time_cap);
716 
717 			if (attr->rx_adv_hibern8_time_cap)
718 				ufshcd_dme_set(hba,
719 					UIC_ARG_MIB_SEL(RX_ADV_HIBERN8TIME_CAP,
720 						i),
721 						attr->rx_adv_hibern8_time_cap);
722 		}
723 	}
724 
725 	exynos_ufs_disable_ov_tm(hba);
726 }
727 
exynos_ufs_establish_connt(struct exynos_ufs * ufs)728 static void exynos_ufs_establish_connt(struct exynos_ufs *ufs)
729 {
730 	struct ufs_hba *hba = ufs->hba;
731 	enum {
732 		DEV_ID		= 0x00,
733 		PEER_DEV_ID	= 0x01,
734 		PEER_CPORT_ID	= 0x00,
735 		TRAFFIC_CLASS	= 0x00,
736 	};
737 
738 	/* allow cport attributes to be set */
739 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_IDLE);
740 
741 	/* local unipro attributes */
742 	ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID), DEV_ID);
743 	ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID_VALID), true);
744 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), PEER_DEV_ID);
745 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERCPORTID), PEER_CPORT_ID);
746 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_CPORTFLAGS), CPORT_DEF_FLAGS);
747 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_TRAFFICCLASS), TRAFFIC_CLASS);
748 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_CONNECTED);
749 }
750 
exynos_ufs_config_smu(struct exynos_ufs * ufs)751 static void exynos_ufs_config_smu(struct exynos_ufs *ufs)
752 {
753 	u32 reg, val;
754 
755 	if (ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE)
756 		return;
757 
758 	exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val);
759 
760 	/* make encryption disabled by default */
761 	reg = ufsp_readl(ufs, UFSPRSECURITY);
762 	ufsp_writel(ufs, reg | NSSMU, UFSPRSECURITY);
763 	ufsp_writel(ufs, 0x0, UFSPSBEGIN0);
764 	ufsp_writel(ufs, 0xffffffff, UFSPSEND0);
765 	ufsp_writel(ufs, 0xff, UFSPSLUN0);
766 	ufsp_writel(ufs, 0xf1, UFSPSCTRL0);
767 
768 	exynos_ufs_auto_ctrl_hcc_restore(ufs, &val);
769 }
770 
exynos_ufs_config_sync_pattern_mask(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)771 static void exynos_ufs_config_sync_pattern_mask(struct exynos_ufs *ufs,
772 					struct ufs_pa_layer_attr *pwr)
773 {
774 	struct ufs_hba *hba = ufs->hba;
775 	u8 g = max_t(u32, pwr->gear_rx, pwr->gear_tx);
776 	u32 mask, sync_len;
777 	enum {
778 		SYNC_LEN_G1 = 80 * 1000, /* 80us */
779 		SYNC_LEN_G2 = 40 * 1000, /* 44us */
780 		SYNC_LEN_G3 = 20 * 1000, /* 20us */
781 	};
782 	int i;
783 
784 	if (g == 1)
785 		sync_len = SYNC_LEN_G1;
786 	else if (g == 2)
787 		sync_len = SYNC_LEN_G2;
788 	else if (g == 3)
789 		sync_len = SYNC_LEN_G3;
790 	else
791 		return;
792 
793 	mask = exynos_ufs_calc_time_cntr(ufs, sync_len);
794 	mask = (mask >> 8) & 0xff;
795 
796 	exynos_ufs_enable_ov_tm(hba);
797 
798 	for_each_ufs_rx_lane(ufs, i)
799 		ufshcd_dme_set(hba,
800 			UIC_ARG_MIB_SEL(RX_SYNC_MASK_LENGTH, i), mask);
801 
802 	exynos_ufs_disable_ov_tm(hba);
803 }
804 
805 #define UFS_HW_VER_MAJOR_MASK   GENMASK(15, 8)
806 
exynos_ufs_get_hs_gear(struct ufs_hba * hba)807 static u32 exynos_ufs_get_hs_gear(struct ufs_hba *hba)
808 {
809 	u8 major;
810 
811 	major = FIELD_GET(UFS_HW_VER_MAJOR_MASK, hba->ufs_version);
812 
813 	if (major >= 3)
814 		return UFS_HS_G4;
815 
816 	/* Default is HS-G3 */
817 	return UFS_HS_G3;
818 }
819 
exynos_ufs_pre_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * dev_max_params,struct ufs_pa_layer_attr * dev_req_params)820 static int exynos_ufs_pre_pwr_mode(struct ufs_hba *hba,
821 				struct ufs_pa_layer_attr *dev_max_params,
822 				struct ufs_pa_layer_attr *dev_req_params)
823 {
824 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
825 	struct phy *generic_phy = ufs->phy;
826 	struct ufs_host_params host_params;
827 	int ret;
828 
829 	if (!dev_req_params) {
830 		pr_err("%s: incoming dev_req_params is NULL\n", __func__);
831 		ret = -EINVAL;
832 		goto out;
833 	}
834 
835 	ufshcd_init_host_params(&host_params);
836 
837 	/* This driver only support symmetric gear setting e.g. hs_tx_gear == hs_rx_gear */
838 	host_params.hs_tx_gear = exynos_ufs_get_hs_gear(hba);
839 	host_params.hs_rx_gear = exynos_ufs_get_hs_gear(hba);
840 
841 	ret = ufshcd_negotiate_pwr_params(&host_params, dev_max_params, dev_req_params);
842 	if (ret) {
843 		pr_err("%s: failed to determine capabilities\n", __func__);
844 		goto out;
845 	}
846 
847 	if (ufs->drv_data->pre_pwr_change)
848 		ufs->drv_data->pre_pwr_change(ufs, dev_req_params);
849 
850 	if (ufshcd_is_hs_mode(dev_req_params)) {
851 		exynos_ufs_config_sync_pattern_mask(ufs, dev_req_params);
852 
853 		switch (dev_req_params->hs_rate) {
854 		case PA_HS_MODE_A:
855 		case PA_HS_MODE_B:
856 			phy_calibrate(generic_phy);
857 			break;
858 		}
859 	}
860 
861 	/* setting for three timeout values for traffic class #0 */
862 	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_FC0PROTTIMEOUTVAL), 8064);
863 	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_TC0REPLAYTIMEOUTVAL), 28224);
864 	ufshcd_dme_set(hba, UIC_ARG_MIB(DL_AFC0REQTIMEOUTVAL), 20160);
865 
866 	return 0;
867 out:
868 	return ret;
869 }
870 
871 #define PWR_MODE_STR_LEN	64
exynos_ufs_post_pwr_mode(struct ufs_hba * hba,struct ufs_pa_layer_attr * pwr_req)872 static int exynos_ufs_post_pwr_mode(struct ufs_hba *hba,
873 				struct ufs_pa_layer_attr *pwr_req)
874 {
875 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
876 	struct phy *generic_phy = ufs->phy;
877 	int gear = max_t(u32, pwr_req->gear_rx, pwr_req->gear_tx);
878 	int lanes = max_t(u32, pwr_req->lane_rx, pwr_req->lane_tx);
879 	char pwr_str[PWR_MODE_STR_LEN] = "";
880 
881 	/* let default be PWM Gear 1, Lane 1 */
882 	if (!gear)
883 		gear = 1;
884 
885 	if (!lanes)
886 		lanes = 1;
887 
888 	if (ufs->drv_data->post_pwr_change)
889 		ufs->drv_data->post_pwr_change(ufs, pwr_req);
890 
891 	if ((ufshcd_is_hs_mode(pwr_req))) {
892 		switch (pwr_req->hs_rate) {
893 		case PA_HS_MODE_A:
894 		case PA_HS_MODE_B:
895 			phy_calibrate(generic_phy);
896 			break;
897 		}
898 
899 		snprintf(pwr_str, PWR_MODE_STR_LEN, "%s series_%s G_%d L_%d",
900 			"FAST",	pwr_req->hs_rate == PA_HS_MODE_A ? "A" : "B",
901 			gear, lanes);
902 	} else {
903 		snprintf(pwr_str, PWR_MODE_STR_LEN, "%s G_%d L_%d",
904 			"SLOW", gear, lanes);
905 	}
906 
907 	dev_info(hba->dev, "Power mode changed to : %s\n", pwr_str);
908 
909 	return 0;
910 }
911 
exynos_ufs_specify_nexus_t_xfer_req(struct ufs_hba * hba,int tag,bool is_scsi_cmd)912 static void exynos_ufs_specify_nexus_t_xfer_req(struct ufs_hba *hba,
913 						int tag, bool is_scsi_cmd)
914 {
915 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
916 	u32 type;
917 
918 	type =  hci_readl(ufs, HCI_UTRL_NEXUS_TYPE);
919 
920 	if (is_scsi_cmd)
921 		hci_writel(ufs, type | (1 << tag), HCI_UTRL_NEXUS_TYPE);
922 	else
923 		hci_writel(ufs, type & ~(1 << tag), HCI_UTRL_NEXUS_TYPE);
924 }
925 
exynos_ufs_specify_nexus_t_tm_req(struct ufs_hba * hba,int tag,u8 func)926 static void exynos_ufs_specify_nexus_t_tm_req(struct ufs_hba *hba,
927 						int tag, u8 func)
928 {
929 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
930 	u32 type;
931 
932 	type =  hci_readl(ufs, HCI_UTMRL_NEXUS_TYPE);
933 
934 	switch (func) {
935 	case UFS_ABORT_TASK:
936 	case UFS_QUERY_TASK:
937 		hci_writel(ufs, type | (1 << tag), HCI_UTMRL_NEXUS_TYPE);
938 		break;
939 	case UFS_ABORT_TASK_SET:
940 	case UFS_CLEAR_TASK_SET:
941 	case UFS_LOGICAL_RESET:
942 	case UFS_QUERY_TASK_SET:
943 		hci_writel(ufs, type & ~(1 << tag), HCI_UTMRL_NEXUS_TYPE);
944 		break;
945 	}
946 }
947 
exynos_ufs_phy_init(struct exynos_ufs * ufs)948 static int exynos_ufs_phy_init(struct exynos_ufs *ufs)
949 {
950 	struct ufs_hba *hba = ufs->hba;
951 	struct phy *generic_phy = ufs->phy;
952 	int ret = 0;
953 
954 	if (ufs->avail_ln_rx == 0 || ufs->avail_ln_tx == 0) {
955 		ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILRXDATALANES),
956 			&ufs->avail_ln_rx);
957 		ufshcd_dme_get(hba, UIC_ARG_MIB(PA_AVAILTXDATALANES),
958 			&ufs->avail_ln_tx);
959 		WARN(ufs->avail_ln_rx != ufs->avail_ln_tx,
960 			"available data lane is not equal(rx:%d, tx:%d)\n",
961 			ufs->avail_ln_rx, ufs->avail_ln_tx);
962 	}
963 
964 	phy_set_bus_width(generic_phy, ufs->avail_ln_rx);
965 
966 	if (generic_phy->power_count) {
967 		phy_power_off(generic_phy);
968 		phy_exit(generic_phy);
969 	}
970 
971 	ret = phy_init(generic_phy);
972 	if (ret) {
973 		dev_err(hba->dev, "%s: phy init failed, ret = %d\n",
974 			__func__, ret);
975 		return ret;
976 	}
977 
978 	ret = phy_power_on(generic_phy);
979 	if (ret)
980 		goto out_exit_phy;
981 
982 	return 0;
983 
984 out_exit_phy:
985 	phy_exit(generic_phy);
986 
987 	return ret;
988 }
989 
exynos_ufs_config_unipro(struct exynos_ufs * ufs)990 static void exynos_ufs_config_unipro(struct exynos_ufs *ufs)
991 {
992 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
993 	struct ufs_hba *hba = ufs->hba;
994 
995 	if (attr->pa_dbg_clk_period_off)
996 		ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_clk_period_off),
997 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
998 
999 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTRAILINGCLOCKS),
1000 			ufs->drv_data->uic_attr->tx_trailingclks);
1001 
1002 	if (attr->pa_dbg_opt_suite1_off)
1003 		ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off),
1004 			       attr->pa_dbg_opt_suite1_val);
1005 
1006 	if (attr->pa_dbg_opt_suite2_off)
1007 		ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite2_off),
1008 			       attr->pa_dbg_opt_suite2_val);
1009 }
1010 
exynos_ufs_config_intr(struct exynos_ufs * ufs,u32 errs,u8 index)1011 static void exynos_ufs_config_intr(struct exynos_ufs *ufs, u32 errs, u8 index)
1012 {
1013 	switch (index) {
1014 	case UNIPRO_L1_5:
1015 		hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_PA_LAYER);
1016 		break;
1017 	case UNIPRO_L2:
1018 		hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DL_LAYER);
1019 		break;
1020 	case UNIPRO_L3:
1021 		hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_N_LAYER);
1022 		break;
1023 	case UNIPRO_L4:
1024 		hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_T_LAYER);
1025 		break;
1026 	case UNIPRO_DME:
1027 		hci_writel(ufs, DFES_ERR_EN | errs, HCI_ERR_EN_DME_LAYER);
1028 		break;
1029 	}
1030 }
1031 
exynos_ufs_setup_clocks(struct ufs_hba * hba,bool on,enum ufs_notify_change_status status)1032 static int exynos_ufs_setup_clocks(struct ufs_hba *hba, bool on,
1033 				   enum ufs_notify_change_status status)
1034 {
1035 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1036 
1037 	if (!ufs)
1038 		return 0;
1039 
1040 	if (on && status == PRE_CHANGE) {
1041 		if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
1042 			exynos_ufs_disable_auto_ctrl_hcc(ufs);
1043 		exynos_ufs_ungate_clks(ufs);
1044 	} else if (!on && status == POST_CHANGE) {
1045 		exynos_ufs_gate_clks(ufs);
1046 		if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
1047 			exynos_ufs_enable_auto_ctrl_hcc(ufs);
1048 	}
1049 
1050 	return 0;
1051 }
1052 
exynos_ufs_pre_link(struct ufs_hba * hba)1053 static int exynos_ufs_pre_link(struct ufs_hba *hba)
1054 {
1055 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1056 
1057 	/* hci */
1058 	exynos_ufs_config_intr(ufs, DFES_DEF_L2_ERRS, UNIPRO_L2);
1059 	exynos_ufs_config_intr(ufs, DFES_DEF_L3_ERRS, UNIPRO_L3);
1060 	exynos_ufs_config_intr(ufs, DFES_DEF_L4_ERRS, UNIPRO_L4);
1061 	exynos_ufs_set_unipro_pclk_div(ufs);
1062 
1063 	/* unipro */
1064 	exynos_ufs_config_unipro(ufs);
1065 
1066 	/* m-phy */
1067 	exynos_ufs_phy_init(ufs);
1068 	if (!(ufs->opts & EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR)) {
1069 		exynos_ufs_config_phy_time_attr(ufs);
1070 		exynos_ufs_config_phy_cap_attr(ufs);
1071 	}
1072 
1073 	exynos_ufs_setup_clocks(hba, true, PRE_CHANGE);
1074 
1075 	if (ufs->drv_data->pre_link)
1076 		ufs->drv_data->pre_link(ufs);
1077 
1078 	return 0;
1079 }
1080 
exynos_ufs_fit_aggr_timeout(struct exynos_ufs * ufs)1081 static void exynos_ufs_fit_aggr_timeout(struct exynos_ufs *ufs)
1082 {
1083 	u32 val;
1084 
1085 	/* Select function clock (mclk) for timer tick */
1086 	if (ufs->opts & EXYNOS_UFS_OPT_TIMER_TICK_SELECT) {
1087 		val = hci_readl(ufs, HCI_V2P1_CTRL);
1088 		val |= IA_TICK_SEL;
1089 		hci_writel(ufs, val, HCI_V2P1_CTRL);
1090 	}
1091 
1092 	val = exynos_ufs_calc_time_cntr(ufs, IATOVAL_NSEC / CNTR_DIV_VAL);
1093 	hci_writel(ufs, val & CNT_VAL_1US_MASK, HCI_1US_TO_CNT_VAL);
1094 }
1095 
exynos_ufs_post_link(struct ufs_hba * hba)1096 static int exynos_ufs_post_link(struct ufs_hba *hba)
1097 {
1098 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1099 	struct phy *generic_phy = ufs->phy;
1100 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
1101 
1102 	exynos_ufs_establish_connt(ufs);
1103 	exynos_ufs_fit_aggr_timeout(ufs);
1104 
1105 	hci_writel(ufs, 0xa, HCI_DATA_REORDER);
1106 	hci_writel(ufs, ilog2(DATA_UNIT_SIZE), HCI_TXPRDT_ENTRY_SIZE);
1107 	hci_writel(ufs, ilog2(DATA_UNIT_SIZE), HCI_RXPRDT_ENTRY_SIZE);
1108 	hci_writel(ufs, (1 << hba->nutrs) - 1, HCI_UTRL_NEXUS_TYPE);
1109 	hci_writel(ufs, (1 << hba->nutmrs) - 1, HCI_UTMRL_NEXUS_TYPE);
1110 	hci_writel(ufs, 0xf, HCI_AXIDMA_RWDATA_BURST_LEN);
1111 
1112 	if (ufs->opts & EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB)
1113 		ufshcd_dme_set(hba,
1114 			UIC_ARG_MIB(T_DBG_SKIP_INIT_HIBERN8_EXIT), true);
1115 
1116 	if (attr->pa_granularity) {
1117 		exynos_ufs_enable_dbg_mode(hba);
1118 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_GRANULARITY),
1119 				attr->pa_granularity);
1120 		exynos_ufs_disable_dbg_mode(hba);
1121 
1122 		if (attr->pa_tactivate)
1123 			ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
1124 					attr->pa_tactivate);
1125 		if (attr->pa_hibern8time &&
1126 		    !(ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER))
1127 			ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
1128 					attr->pa_hibern8time);
1129 	}
1130 
1131 	if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) {
1132 		if (!attr->pa_granularity)
1133 			ufshcd_dme_get(hba, UIC_ARG_MIB(PA_GRANULARITY),
1134 					&attr->pa_granularity);
1135 		if (!attr->pa_hibern8time)
1136 			ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
1137 					&attr->pa_hibern8time);
1138 		/*
1139 		 * not wait for HIBERN8 time to exit hibernation
1140 		 */
1141 		ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), 0);
1142 
1143 		if (attr->pa_granularity < 1 || attr->pa_granularity > 6) {
1144 			/* Valid range for granularity: 1 ~ 6 */
1145 			dev_warn(hba->dev,
1146 				"%s: pa_granularity %d is invalid, assuming backwards compatibility\n",
1147 				__func__,
1148 				attr->pa_granularity);
1149 			attr->pa_granularity = 6;
1150 		}
1151 	}
1152 
1153 	phy_calibrate(generic_phy);
1154 
1155 	if (ufs->drv_data->post_link)
1156 		ufs->drv_data->post_link(ufs);
1157 
1158 	return 0;
1159 }
1160 
exynos_ufs_parse_dt(struct device * dev,struct exynos_ufs * ufs)1161 static int exynos_ufs_parse_dt(struct device *dev, struct exynos_ufs *ufs)
1162 {
1163 	struct device_node *np = dev->of_node;
1164 	struct exynos_ufs_uic_attr *attr;
1165 	int ret = 0;
1166 
1167 	ufs->drv_data = device_get_match_data(dev);
1168 
1169 	if (ufs->drv_data && ufs->drv_data->uic_attr) {
1170 		attr = ufs->drv_data->uic_attr;
1171 	} else {
1172 		dev_err(dev, "failed to get uic attributes\n");
1173 		ret = -EINVAL;
1174 		goto out;
1175 	}
1176 
1177 	ufs->sysreg = syscon_regmap_lookup_by_phandle(np, "samsung,sysreg");
1178 	if (IS_ERR(ufs->sysreg))
1179 		ufs->sysreg = NULL;
1180 	else {
1181 		if (of_property_read_u32_index(np, "samsung,sysreg", 1,
1182 					       &ufs->iocc_offset)) {
1183 			dev_warn(dev, "can't get an offset from sysreg. Set to default value\n");
1184 			ufs->iocc_offset = UFS_SHAREABILITY_OFFSET;
1185 		}
1186 	}
1187 
1188 	ufs->iocc_mask = ufs->drv_data->iocc_mask;
1189 	/*
1190 	 * no 'dma-coherent' property means the descriptors are
1191 	 * non-cacheable so iocc shareability should be disabled.
1192 	 */
1193 	if (of_dma_is_coherent(dev->of_node))
1194 		ufs->iocc_val = ufs->iocc_mask;
1195 	else
1196 		ufs->iocc_val = 0;
1197 
1198 	ufs->pclk_avail_min = PCLK_AVAIL_MIN;
1199 	ufs->pclk_avail_max = PCLK_AVAIL_MAX;
1200 
1201 	attr->rx_adv_fine_gran_sup_en = RX_ADV_FINE_GRAN_SUP_EN;
1202 	attr->rx_adv_fine_gran_step = RX_ADV_FINE_GRAN_STEP_VAL;
1203 	attr->rx_adv_min_actv_time_cap = RX_ADV_MIN_ACTV_TIME_CAP;
1204 	attr->pa_granularity = PA_GRANULARITY_VAL;
1205 	attr->pa_tactivate = PA_TACTIVATE_VAL;
1206 	attr->pa_hibern8time = PA_HIBERN8TIME_VAL;
1207 
1208 out:
1209 	return ret;
1210 }
1211 
exynos_ufs_priv_init(struct ufs_hba * hba,struct exynos_ufs * ufs)1212 static inline void exynos_ufs_priv_init(struct ufs_hba *hba,
1213 					struct exynos_ufs *ufs)
1214 {
1215 	ufs->hba = hba;
1216 	ufs->opts = ufs->drv_data->opts;
1217 	ufs->rx_sel_idx = PA_MAXDATALANES;
1218 	if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX)
1219 		ufs->rx_sel_idx = 0;
1220 	hba->priv = (void *)ufs;
1221 	hba->quirks = ufs->drv_data->quirks;
1222 }
1223 
1224 #ifdef CONFIG_SCSI_UFS_CRYPTO
1225 
1226 /*
1227  * Support for Flash Memory Protector (FMP), which is the inline encryption
1228  * hardware on Exynos and Exynos-based SoCs.  The interface to this hardware is
1229  * not compatible with the standard UFS crypto.  It requires that encryption be
1230  * configured in the PRDT using a nonstandard extension.
1231  */
1232 
1233 enum fmp_crypto_algo_mode {
1234 	FMP_BYPASS_MODE = 0,
1235 	FMP_ALGO_MODE_AES_CBC = 1,
1236 	FMP_ALGO_MODE_AES_XTS = 2,
1237 };
1238 enum fmp_crypto_key_length {
1239 	FMP_KEYLEN_256BIT = 1,
1240 };
1241 
1242 /**
1243  * struct fmp_sg_entry - nonstandard format of PRDT entries when FMP is enabled
1244  *
1245  * @base: The standard PRDT entry, but with nonstandard bitfields in the high
1246  *	bits of the 'size' field, i.e. the last 32-bit word.  When these
1247  *	nonstandard bitfields are zero, the data segment won't be encrypted or
1248  *	decrypted.  Otherwise they specify the algorithm and key length with
1249  *	which the data segment will be encrypted or decrypted.
1250  * @file_iv: The initialization vector (IV) with all bytes reversed
1251  * @file_enckey: The first half of the AES-XTS key with all bytes reserved
1252  * @file_twkey: The second half of the AES-XTS key with all bytes reserved
1253  * @disk_iv: Unused
1254  * @reserved: Unused
1255  */
1256 struct fmp_sg_entry {
1257 	struct ufshcd_sg_entry base;
1258 	__be64 file_iv[2];
1259 	__be64 file_enckey[4];
1260 	__be64 file_twkey[4];
1261 	__be64 disk_iv[2];
1262 	__be64 reserved[2];
1263 };
1264 
1265 #define SMC_CMD_FMP_SECURITY	\
1266 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \
1267 			   ARM_SMCCC_OWNER_SIP, 0x1810)
1268 #define SMC_CMD_SMU		\
1269 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \
1270 			   ARM_SMCCC_OWNER_SIP, 0x1850)
1271 #define SMC_CMD_FMP_SMU_RESUME	\
1272 	ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, ARM_SMCCC_SMC_64, \
1273 			   ARM_SMCCC_OWNER_SIP, 0x1860)
1274 #define SMU_EMBEDDED			0
1275 #define SMU_INIT			0
1276 #define CFG_DESCTYPE_3			3
1277 
exynos_ufs_fmp_init(struct ufs_hba * hba,struct exynos_ufs * ufs)1278 static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs)
1279 {
1280 	struct blk_crypto_profile *profile = &hba->crypto_profile;
1281 	struct arm_smccc_res res;
1282 	int err;
1283 
1284 	/*
1285 	 * Check for the standard crypto support bit, since it's available even
1286 	 * though the rest of the interface to FMP is nonstandard.
1287 	 *
1288 	 * This check should have the effect of preventing the driver from
1289 	 * trying to use FMP on old Exynos SoCs that don't have FMP.
1290 	 */
1291 	if (!(ufshcd_readl(hba, REG_CONTROLLER_CAPABILITIES) &
1292 	      MASK_CRYPTO_SUPPORT))
1293 		return;
1294 
1295 	/*
1296 	 * The below sequence of SMC calls to enable FMP can be found in the
1297 	 * downstream driver source for gs101 and other Exynos-based SoCs.  It
1298 	 * is the only way to enable FMP that works on SoCs such as gs101 that
1299 	 * don't make the FMP registers accessible to Linux.  It probably works
1300 	 * on other Exynos-based SoCs too, and might even still be the only way
1301 	 * that works.  But this hasn't been properly tested, and this code is
1302 	 * mutually exclusive with exynos_ufs_config_smu().  So for now only
1303 	 * enable FMP support on SoCs with EXYNOS_UFS_OPT_UFSPR_SECURE.
1304 	 */
1305 	if (!(ufs->opts & EXYNOS_UFS_OPT_UFSPR_SECURE))
1306 		return;
1307 
1308 	/*
1309 	 * This call (which sets DESCTYPE to 0x3 in the FMPSECURITY0 register)
1310 	 * is needed to make the hardware use the larger PRDT entry size.
1311 	 */
1312 	BUILD_BUG_ON(sizeof(struct fmp_sg_entry) != 128);
1313 	arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3,
1314 		      0, 0, 0, 0, &res);
1315 	if (res.a0) {
1316 		dev_warn(hba->dev,
1317 			 "SMC_CMD_FMP_SECURITY failed on init: %ld.  Disabling FMP support.\n",
1318 			 res.a0);
1319 		return;
1320 	}
1321 	ufshcd_set_sg_entry_size(hba, sizeof(struct fmp_sg_entry));
1322 
1323 	/*
1324 	 * This is needed to initialize FMP.  Without it, errors occur when
1325 	 * inline encryption is used.
1326 	 */
1327 	arm_smccc_smc(SMC_CMD_SMU, SMU_INIT, SMU_EMBEDDED, 0, 0, 0, 0, 0, &res);
1328 	if (res.a0) {
1329 		dev_err(hba->dev,
1330 			"SMC_CMD_SMU(SMU_INIT) failed: %ld.  Disabling FMP support.\n",
1331 			res.a0);
1332 		return;
1333 	}
1334 
1335 	/* Advertise crypto capabilities to the block layer. */
1336 	err = devm_blk_crypto_profile_init(hba->dev, profile, 0);
1337 	if (err) {
1338 		/* Only ENOMEM should be possible here. */
1339 		dev_err(hba->dev, "Failed to initialize crypto profile: %d\n",
1340 			err);
1341 		return;
1342 	}
1343 	profile->max_dun_bytes_supported = AES_BLOCK_SIZE;
1344 	profile->dev = hba->dev;
1345 	profile->modes_supported[BLK_ENCRYPTION_MODE_AES_256_XTS] =
1346 		DATA_UNIT_SIZE;
1347 
1348 	/* Advertise crypto support to ufshcd-core. */
1349 	hba->caps |= UFSHCD_CAP_CRYPTO;
1350 
1351 	/* Advertise crypto quirks to ufshcd-core. */
1352 	hba->quirks |= UFSHCD_QUIRK_CUSTOM_CRYPTO_PROFILE |
1353 		       UFSHCD_QUIRK_BROKEN_CRYPTO_ENABLE |
1354 		       UFSHCD_QUIRK_KEYS_IN_PRDT;
1355 
1356 }
1357 
exynos_ufs_fmp_resume(struct ufs_hba * hba)1358 static void exynos_ufs_fmp_resume(struct ufs_hba *hba)
1359 {
1360 	struct arm_smccc_res res;
1361 
1362 	if (!(hba->caps & UFSHCD_CAP_CRYPTO))
1363 		return;
1364 
1365 	arm_smccc_smc(SMC_CMD_FMP_SECURITY, 0, SMU_EMBEDDED, CFG_DESCTYPE_3,
1366 		      0, 0, 0, 0, &res);
1367 	if (res.a0)
1368 		dev_err(hba->dev,
1369 			"SMC_CMD_FMP_SECURITY failed on resume: %ld\n", res.a0);
1370 
1371 	arm_smccc_smc(SMC_CMD_FMP_SMU_RESUME, 0, SMU_EMBEDDED, 0, 0, 0, 0, 0,
1372 		      &res);
1373 	if (res.a0)
1374 		dev_err(hba->dev,
1375 			"SMC_CMD_FMP_SMU_RESUME failed: %ld\n", res.a0);
1376 }
1377 
fmp_key_word(const u8 * key,int j)1378 static inline __be64 fmp_key_word(const u8 *key, int j)
1379 {
1380 	return cpu_to_be64(get_unaligned_le64(
1381 			key + AES_KEYSIZE_256 - (j + 1) * sizeof(u64)));
1382 }
1383 
1384 /* Fill the PRDT for a request according to the given encryption context. */
exynos_ufs_fmp_fill_prdt(struct ufs_hba * hba,const struct bio_crypt_ctx * crypt_ctx,void * prdt,unsigned int num_segments)1385 static int exynos_ufs_fmp_fill_prdt(struct ufs_hba *hba,
1386 				    const struct bio_crypt_ctx *crypt_ctx,
1387 				    void *prdt, unsigned int num_segments)
1388 {
1389 	struct fmp_sg_entry *fmp_prdt = prdt;
1390 	const u8 *enckey = crypt_ctx->bc_key->raw;
1391 	const u8 *twkey = enckey + AES_KEYSIZE_256;
1392 	u64 dun_lo = crypt_ctx->bc_dun[0];
1393 	u64 dun_hi = crypt_ctx->bc_dun[1];
1394 	unsigned int i;
1395 
1396 	/* If FMP wasn't enabled, we shouldn't get any encrypted requests. */
1397 	if (WARN_ON_ONCE(!(hba->caps & UFSHCD_CAP_CRYPTO)))
1398 		return -EIO;
1399 
1400 	/* Configure FMP on each segment of the request. */
1401 	for (i = 0; i < num_segments; i++) {
1402 		struct fmp_sg_entry *prd = &fmp_prdt[i];
1403 		int j;
1404 
1405 		/* Each segment must be exactly one data unit. */
1406 		if (prd->base.size != cpu_to_le32(DATA_UNIT_SIZE - 1)) {
1407 			dev_err(hba->dev,
1408 				"data segment is misaligned for FMP\n");
1409 			return -EIO;
1410 		}
1411 
1412 		/* Set the algorithm and key length. */
1413 		prd->base.size |= cpu_to_le32((FMP_ALGO_MODE_AES_XTS << 28) |
1414 					      (FMP_KEYLEN_256BIT << 26));
1415 
1416 		/* Set the IV. */
1417 		prd->file_iv[0] = cpu_to_be64(dun_hi);
1418 		prd->file_iv[1] = cpu_to_be64(dun_lo);
1419 
1420 		/* Set the key. */
1421 		for (j = 0; j < AES_KEYSIZE_256 / sizeof(u64); j++) {
1422 			prd->file_enckey[j] = fmp_key_word(enckey, j);
1423 			prd->file_twkey[j] = fmp_key_word(twkey, j);
1424 		}
1425 
1426 		/* Increment the data unit number. */
1427 		dun_lo++;
1428 		if (dun_lo == 0)
1429 			dun_hi++;
1430 	}
1431 	return 0;
1432 }
1433 
1434 #else /* CONFIG_SCSI_UFS_CRYPTO */
1435 
exynos_ufs_fmp_init(struct ufs_hba * hba,struct exynos_ufs * ufs)1436 static void exynos_ufs_fmp_init(struct ufs_hba *hba, struct exynos_ufs *ufs)
1437 {
1438 }
1439 
exynos_ufs_fmp_resume(struct ufs_hba * hba)1440 static void exynos_ufs_fmp_resume(struct ufs_hba *hba)
1441 {
1442 }
1443 
1444 #define exynos_ufs_fmp_fill_prdt NULL
1445 
1446 #endif /* !CONFIG_SCSI_UFS_CRYPTO */
1447 
exynos_ufs_init(struct ufs_hba * hba)1448 static int exynos_ufs_init(struct ufs_hba *hba)
1449 {
1450 	struct device *dev = hba->dev;
1451 	struct platform_device *pdev = to_platform_device(dev);
1452 	struct exynos_ufs *ufs;
1453 	int ret;
1454 
1455 	ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL);
1456 	if (!ufs)
1457 		return -ENOMEM;
1458 
1459 	/* exynos-specific hci */
1460 	ufs->reg_hci = devm_platform_ioremap_resource_byname(pdev, "vs_hci");
1461 	if (IS_ERR(ufs->reg_hci)) {
1462 		dev_err(dev, "cannot ioremap for hci vendor register\n");
1463 		return PTR_ERR(ufs->reg_hci);
1464 	}
1465 
1466 	/* unipro */
1467 	ufs->reg_unipro = devm_platform_ioremap_resource_byname(pdev, "unipro");
1468 	if (IS_ERR(ufs->reg_unipro)) {
1469 		dev_err(dev, "cannot ioremap for unipro register\n");
1470 		return PTR_ERR(ufs->reg_unipro);
1471 	}
1472 
1473 	/* ufs protector */
1474 	ufs->reg_ufsp = devm_platform_ioremap_resource_byname(pdev, "ufsp");
1475 	if (IS_ERR(ufs->reg_ufsp)) {
1476 		dev_err(dev, "cannot ioremap for ufs protector register\n");
1477 		return PTR_ERR(ufs->reg_ufsp);
1478 	}
1479 
1480 	ret = exynos_ufs_parse_dt(dev, ufs);
1481 	if (ret) {
1482 		dev_err(dev, "failed to get dt info.\n");
1483 		goto out;
1484 	}
1485 
1486 	ufs->phy = devm_phy_get(dev, "ufs-phy");
1487 	if (IS_ERR(ufs->phy)) {
1488 		ret = PTR_ERR(ufs->phy);
1489 		dev_err(dev, "failed to get ufs-phy\n");
1490 		goto out;
1491 	}
1492 
1493 	exynos_ufs_priv_init(hba, ufs);
1494 
1495 	exynos_ufs_fmp_init(hba, ufs);
1496 
1497 	if (ufs->drv_data->drv_init) {
1498 		ret = ufs->drv_data->drv_init(ufs);
1499 		if (ret) {
1500 			dev_err(dev, "failed to init drv-data\n");
1501 			goto out;
1502 		}
1503 	}
1504 
1505 	ret = exynos_ufs_get_clk_info(ufs);
1506 	if (ret)
1507 		goto out;
1508 	exynos_ufs_specify_phy_time_attr(ufs);
1509 
1510 	exynos_ufs_config_smu(ufs);
1511 
1512 	hba->host->dma_alignment = DATA_UNIT_SIZE - 1;
1513 	return 0;
1514 
1515 out:
1516 	hba->priv = NULL;
1517 	return ret;
1518 }
1519 
exynos_ufs_host_reset(struct ufs_hba * hba)1520 static int exynos_ufs_host_reset(struct ufs_hba *hba)
1521 {
1522 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1523 	unsigned long timeout = jiffies + msecs_to_jiffies(1);
1524 	u32 val;
1525 	int ret = 0;
1526 
1527 	exynos_ufs_disable_auto_ctrl_hcc_save(ufs, &val);
1528 
1529 	hci_writel(ufs, UFS_SW_RST_MASK, HCI_SW_RST);
1530 
1531 	do {
1532 		if (!(hci_readl(ufs, HCI_SW_RST) & UFS_SW_RST_MASK))
1533 			goto out;
1534 	} while (time_before(jiffies, timeout));
1535 
1536 	dev_err(hba->dev, "timeout host sw-reset\n");
1537 	ret = -ETIMEDOUT;
1538 
1539 out:
1540 	exynos_ufs_auto_ctrl_hcc_restore(ufs, &val);
1541 	return ret;
1542 }
1543 
exynos_ufs_dev_hw_reset(struct ufs_hba * hba)1544 static void exynos_ufs_dev_hw_reset(struct ufs_hba *hba)
1545 {
1546 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1547 
1548 	hci_writel(ufs, 0 << 0, HCI_GPIO_OUT);
1549 	udelay(5);
1550 	hci_writel(ufs, 1 << 0, HCI_GPIO_OUT);
1551 }
1552 
exynos_ufs_pre_hibern8(struct ufs_hba * hba,enum uic_cmd_dme cmd)1553 static void exynos_ufs_pre_hibern8(struct ufs_hba *hba, enum uic_cmd_dme cmd)
1554 {
1555 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1556 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
1557 
1558 	if (cmd == UIC_CMD_DME_HIBER_EXIT) {
1559 		if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
1560 			exynos_ufs_disable_auto_ctrl_hcc(ufs);
1561 		exynos_ufs_ungate_clks(ufs);
1562 
1563 		if (ufs->opts & EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER) {
1564 			static const unsigned int granularity_tbl[] = {
1565 				1, 4, 8, 16, 32, 100
1566 			};
1567 			int h8_time = attr->pa_hibern8time *
1568 				granularity_tbl[attr->pa_granularity - 1];
1569 			unsigned long us;
1570 			s64 delta;
1571 
1572 			do {
1573 				delta = h8_time - ktime_us_delta(ktime_get(),
1574 							ufs->entry_hibern8_t);
1575 				if (delta <= 0)
1576 					break;
1577 
1578 				us = min_t(s64, delta, USEC_PER_MSEC);
1579 				if (us >= 10)
1580 					usleep_range(us, us + 10);
1581 			} while (1);
1582 		}
1583 	}
1584 }
1585 
exynos_ufs_post_hibern8(struct ufs_hba * hba,enum uic_cmd_dme cmd)1586 static void exynos_ufs_post_hibern8(struct ufs_hba *hba, enum uic_cmd_dme cmd)
1587 {
1588 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1589 
1590 	if (cmd == UIC_CMD_DME_HIBER_ENTER) {
1591 		ufs->entry_hibern8_t = ktime_get();
1592 		exynos_ufs_gate_clks(ufs);
1593 		if (ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL)
1594 			exynos_ufs_enable_auto_ctrl_hcc(ufs);
1595 	}
1596 }
1597 
exynos_ufs_hce_enable_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)1598 static int exynos_ufs_hce_enable_notify(struct ufs_hba *hba,
1599 					enum ufs_notify_change_status status)
1600 {
1601 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1602 	int ret = 0;
1603 
1604 	switch (status) {
1605 	case PRE_CHANGE:
1606 		/*
1607 		 * The maximum segment size must be set after scsi_host_alloc()
1608 		 * has been called and before LUN scanning starts
1609 		 * (ufshcd_async_scan()). Note: this callback may also be called
1610 		 * from other functions than ufshcd_init().
1611 		 */
1612 		hba->host->max_segment_size = DATA_UNIT_SIZE;
1613 
1614 		if (ufs->drv_data->pre_hce_enable) {
1615 			ret = ufs->drv_data->pre_hce_enable(ufs);
1616 			if (ret)
1617 				return ret;
1618 		}
1619 
1620 		ret = exynos_ufs_host_reset(hba);
1621 		if (ret)
1622 			return ret;
1623 		exynos_ufs_dev_hw_reset(hba);
1624 		break;
1625 	case POST_CHANGE:
1626 		exynos_ufs_calc_pwm_clk_div(ufs);
1627 		if (!(ufs->opts & EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL))
1628 			exynos_ufs_enable_auto_ctrl_hcc(ufs);
1629 
1630 		if (ufs->drv_data->post_hce_enable)
1631 			ret = ufs->drv_data->post_hce_enable(ufs);
1632 
1633 		break;
1634 	}
1635 
1636 	return ret;
1637 }
1638 
exynos_ufs_link_startup_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)1639 static int exynos_ufs_link_startup_notify(struct ufs_hba *hba,
1640 					  enum ufs_notify_change_status status)
1641 {
1642 	int ret = 0;
1643 
1644 	switch (status) {
1645 	case PRE_CHANGE:
1646 		ret = exynos_ufs_pre_link(hba);
1647 		break;
1648 	case POST_CHANGE:
1649 		ret = exynos_ufs_post_link(hba);
1650 		break;
1651 	}
1652 
1653 	return ret;
1654 }
1655 
exynos_ufs_pwr_change_notify(struct ufs_hba * hba,enum ufs_notify_change_status status,struct ufs_pa_layer_attr * dev_max_params,struct ufs_pa_layer_attr * dev_req_params)1656 static int exynos_ufs_pwr_change_notify(struct ufs_hba *hba,
1657 				enum ufs_notify_change_status status,
1658 				struct ufs_pa_layer_attr *dev_max_params,
1659 				struct ufs_pa_layer_attr *dev_req_params)
1660 {
1661 	int ret = 0;
1662 
1663 	switch (status) {
1664 	case PRE_CHANGE:
1665 		ret = exynos_ufs_pre_pwr_mode(hba, dev_max_params,
1666 					      dev_req_params);
1667 		break;
1668 	case POST_CHANGE:
1669 		ret = exynos_ufs_post_pwr_mode(hba, dev_req_params);
1670 		break;
1671 	}
1672 
1673 	return ret;
1674 }
1675 
exynos_ufs_hibern8_notify(struct ufs_hba * hba,enum uic_cmd_dme cmd,enum ufs_notify_change_status notify)1676 static void exynos_ufs_hibern8_notify(struct ufs_hba *hba,
1677 				     enum uic_cmd_dme cmd,
1678 				     enum ufs_notify_change_status notify)
1679 {
1680 	switch ((u8)notify) {
1681 	case PRE_CHANGE:
1682 		exynos_ufs_pre_hibern8(hba, cmd);
1683 		break;
1684 	case POST_CHANGE:
1685 		exynos_ufs_post_hibern8(hba, cmd);
1686 		break;
1687 	}
1688 }
1689 
exynos_ufs_suspend(struct ufs_hba * hba,enum ufs_pm_op pm_op,enum ufs_notify_change_status status)1690 static int exynos_ufs_suspend(struct ufs_hba *hba, enum ufs_pm_op pm_op,
1691 	enum ufs_notify_change_status status)
1692 {
1693 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1694 
1695 	if (status == PRE_CHANGE)
1696 		return 0;
1697 
1698 	if (!ufshcd_is_link_active(hba))
1699 		phy_power_off(ufs->phy);
1700 
1701 	return 0;
1702 }
1703 
exynos_ufs_resume(struct ufs_hba * hba,enum ufs_pm_op pm_op)1704 static int exynos_ufs_resume(struct ufs_hba *hba, enum ufs_pm_op pm_op)
1705 {
1706 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
1707 
1708 	if (!ufshcd_is_link_active(hba))
1709 		phy_power_on(ufs->phy);
1710 
1711 	exynos_ufs_config_smu(ufs);
1712 	exynos_ufs_fmp_resume(hba);
1713 	return 0;
1714 }
1715 
exynosauto_ufs_vh_link_startup_notify(struct ufs_hba * hba,enum ufs_notify_change_status status)1716 static int exynosauto_ufs_vh_link_startup_notify(struct ufs_hba *hba,
1717 						 enum ufs_notify_change_status status)
1718 {
1719 	if (status == POST_CHANGE) {
1720 		ufshcd_set_link_active(hba);
1721 		ufshcd_set_ufs_dev_active(hba);
1722 	}
1723 
1724 	return 0;
1725 }
1726 
exynosauto_ufs_vh_wait_ph_ready(struct ufs_hba * hba)1727 static int exynosauto_ufs_vh_wait_ph_ready(struct ufs_hba *hba)
1728 {
1729 	u32 mbox;
1730 	ktime_t start, stop;
1731 
1732 	start = ktime_get();
1733 	stop = ktime_add(start, ms_to_ktime(PH_READY_TIMEOUT_MS));
1734 
1735 	do {
1736 		mbox = ufshcd_readl(hba, PH2VH_MBOX);
1737 		/* TODO: Mailbox message protocols between the PH and VHs are
1738 		 * not implemented yet. This will be supported later
1739 		 */
1740 		if ((mbox & MH_MSG_MASK) == MH_MSG_PH_READY)
1741 			return 0;
1742 
1743 		usleep_range(40, 50);
1744 	} while (ktime_before(ktime_get(), stop));
1745 
1746 	return -ETIME;
1747 }
1748 
exynosauto_ufs_vh_init(struct ufs_hba * hba)1749 static int exynosauto_ufs_vh_init(struct ufs_hba *hba)
1750 {
1751 	struct device *dev = hba->dev;
1752 	struct platform_device *pdev = to_platform_device(dev);
1753 	struct exynos_ufs *ufs;
1754 	int ret;
1755 
1756 	ufs = devm_kzalloc(dev, sizeof(*ufs), GFP_KERNEL);
1757 	if (!ufs)
1758 		return -ENOMEM;
1759 
1760 	/* exynos-specific hci */
1761 	ufs->reg_hci = devm_platform_ioremap_resource_byname(pdev, "vs_hci");
1762 	if (IS_ERR(ufs->reg_hci)) {
1763 		dev_err(dev, "cannot ioremap for hci vendor register\n");
1764 		return PTR_ERR(ufs->reg_hci);
1765 	}
1766 
1767 	ret = exynosauto_ufs_vh_wait_ph_ready(hba);
1768 	if (ret)
1769 		return ret;
1770 
1771 	ufs->drv_data = device_get_match_data(dev);
1772 	if (!ufs->drv_data)
1773 		return -ENODEV;
1774 
1775 	exynos_ufs_priv_init(hba, ufs);
1776 
1777 	return 0;
1778 }
1779 
fsd_ufs_pre_link(struct exynos_ufs * ufs)1780 static int fsd_ufs_pre_link(struct exynos_ufs *ufs)
1781 {
1782 	struct exynos_ufs_uic_attr *attr = ufs->drv_data->uic_attr;
1783 	struct ufs_hba *hba = ufs->hba;
1784 	int i;
1785 
1786 	ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_clk_period_off),
1787 		       DIV_ROUND_UP(NSEC_PER_SEC,  ufs->mclk_rate));
1788 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x201), 0x12);
1789 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40);
1790 
1791 	for_each_ufs_tx_lane(ufs, i) {
1792 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0xAA, i),
1793 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
1794 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x8F, i), 0x3F);
1795 	}
1796 
1797 	for_each_ufs_rx_lane(ufs, i) {
1798 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x12, i),
1799 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
1800 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x5C, i), 0x38);
1801 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x0F, i), 0x0);
1802 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x65, i), 0x1);
1803 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x69, i), 0x1);
1804 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x21, i), 0x0);
1805 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x22, i), 0x0);
1806 	}
1807 
1808 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0);
1809 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_AUTOMODE_THLD), 0x4E20);
1810 
1811 	ufshcd_dme_set(hba, UIC_ARG_MIB(attr->pa_dbg_opt_suite1_off),
1812 		       0x2e820183);
1813 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0);
1814 
1815 	exynos_ufs_establish_connt(ufs);
1816 
1817 	return 0;
1818 }
1819 
fsd_ufs_post_link(struct exynos_ufs * ufs)1820 static int fsd_ufs_post_link(struct exynos_ufs *ufs)
1821 {
1822 	int i;
1823 	struct ufs_hba *hba = ufs->hba;
1824 	u32 hw_cap_min_tactivate;
1825 	u32 peer_rx_min_actv_time_cap;
1826 	u32 max_rx_hibern8_time_cap;
1827 
1828 	ufshcd_dme_get(hba, UIC_ARG_MIB_SEL(0x8F, 4),
1829 			&hw_cap_min_tactivate); /* HW Capability of MIN_TACTIVATE */
1830 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_TACTIVATE),
1831 			&peer_rx_min_actv_time_cap);    /* PA_TActivate */
1832 	ufshcd_dme_get(hba, UIC_ARG_MIB(PA_HIBERN8TIME),
1833 			&max_rx_hibern8_time_cap);      /* PA_Hibern8Time */
1834 
1835 	if (peer_rx_min_actv_time_cap >= hw_cap_min_tactivate)
1836 		ufshcd_dme_peer_set(hba, UIC_ARG_MIB(PA_TACTIVATE),
1837 					peer_rx_min_actv_time_cap + 1);
1838 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_HIBERN8TIME), max_rx_hibern8_time_cap + 1);
1839 
1840 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), 0x01);
1841 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0xFA);
1842 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_DBG_MODE), 0x00);
1843 
1844 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40);
1845 
1846 	for_each_ufs_rx_lane(ufs, i) {
1847 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x35, i), 0x05);
1848 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x73, i), 0x01);
1849 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x41, i), 0x02);
1850 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x42, i), 0xAC);
1851 	}
1852 
1853 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0);
1854 
1855 	return 0;
1856 }
1857 
fsd_ufs_pre_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)1858 static int fsd_ufs_pre_pwr_change(struct exynos_ufs *ufs,
1859 					struct ufs_pa_layer_attr *pwr)
1860 {
1861 	struct ufs_hba *hba = ufs->hba;
1862 
1863 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_TXTERMINATION), 0x1);
1864 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_RXTERMINATION), 0x1);
1865 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000);
1866 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000);
1867 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000);
1868 
1869 	unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0);
1870 	unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1);
1871 	unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2);
1872 
1873 	return 0;
1874 }
1875 
get_mclk_period_unipro_18(struct exynos_ufs * ufs)1876 static inline u32 get_mclk_period_unipro_18(struct exynos_ufs *ufs)
1877 {
1878 	return (16 * 1000 * 1000000UL / ufs->mclk_rate);
1879 }
1880 
gs101_ufs_pre_link(struct exynos_ufs * ufs)1881 static int gs101_ufs_pre_link(struct exynos_ufs *ufs)
1882 {
1883 	struct ufs_hba *hba = ufs->hba;
1884 	int i;
1885 	u32 tx_line_reset_period, rx_line_reset_period;
1886 
1887 	rx_line_reset_period = (RX_LINE_RESET_TIME * ufs->mclk_rate)
1888 				/ NSEC_PER_MSEC;
1889 	tx_line_reset_period = (TX_LINE_RESET_TIME * ufs->mclk_rate)
1890 				/ NSEC_PER_MSEC;
1891 
1892 	unipro_writel(ufs, get_mclk_period_unipro_18(ufs), COMP_CLK_PERIOD);
1893 
1894 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x40);
1895 
1896 	for_each_ufs_rx_lane(ufs, i) {
1897 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD, i),
1898 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
1899 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_CLK_PRD_EN, i), 0x0);
1900 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE2, i),
1901 			       (rx_line_reset_period >> 16) & 0xFF);
1902 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE1, i),
1903 			       (rx_line_reset_period >> 8) & 0xFF);
1904 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_RX_LINERESET_VALUE0, i),
1905 			       (rx_line_reset_period) & 0xFF);
1906 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x2f, i), 0x69);
1907 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x84, i), 0x1);
1908 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x25, i), 0xf6);
1909 	}
1910 
1911 	for_each_ufs_tx_lane(ufs, i) {
1912 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD, i),
1913 			       DIV_ROUND_UP(NSEC_PER_SEC, ufs->mclk_rate));
1914 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_CLK_PRD_EN, i),
1915 			       0x02);
1916 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE2, i),
1917 			       (tx_line_reset_period >> 16) & 0xFF);
1918 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE1, i),
1919 			       (tx_line_reset_period >> 8) & 0xFF);
1920 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(VND_TX_LINERESET_PVALUE0, i),
1921 			       (tx_line_reset_period) & 0xFF);
1922 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x04, i), 1);
1923 		ufshcd_dme_set(hba, UIC_ARG_MIB_SEL(0x7F, i), 0);
1924 	}
1925 
1926 	ufshcd_dme_set(hba, UIC_ARG_MIB(0x200), 0x0);
1927 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_LOCAL_TX_LCC_ENABLE), 0x0);
1928 	ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID), 0x0);
1929 	ufshcd_dme_set(hba, UIC_ARG_MIB(N_DEVICEID_VALID), 0x1);
1930 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_PEERDEVICEID), 0x1);
1931 	ufshcd_dme_set(hba, UIC_ARG_MIB(T_CONNECTIONSTATE), CPORT_CONNECTED);
1932 	ufshcd_dme_set(hba, UIC_ARG_MIB(0xA006), 0x8000);
1933 
1934 	return 0;
1935 }
1936 
gs101_ufs_post_link(struct exynos_ufs * ufs)1937 static int gs101_ufs_post_link(struct exynos_ufs *ufs)
1938 {
1939 	struct ufs_hba *hba = ufs->hba;
1940 
1941 	/*
1942 	 * Enable Write Line Unique. This field has to be 0x3
1943 	 * to support Write Line Unique transaction on gs101.
1944 	 */
1945 	hci_writel(ufs, WLU_EN | WLU_BURST_LEN(3), HCI_AXIDMA_RWDATA_BURST_LEN);
1946 
1947 	exynos_ufs_enable_dbg_mode(hba);
1948 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_SAVECONFIGTIME), 0x3e8);
1949 	exynos_ufs_disable_dbg_mode(hba);
1950 
1951 	return 0;
1952 }
1953 
gs101_ufs_pre_pwr_change(struct exynos_ufs * ufs,struct ufs_pa_layer_attr * pwr)1954 static int gs101_ufs_pre_pwr_change(struct exynos_ufs *ufs,
1955 					 struct ufs_pa_layer_attr *pwr)
1956 {
1957 	struct ufs_hba *hba = ufs->hba;
1958 
1959 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA0), 12000);
1960 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA1), 32000);
1961 	ufshcd_dme_set(hba, UIC_ARG_MIB(PA_PWRMODEUSERDATA2), 16000);
1962 	unipro_writel(ufs, 8064, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER0);
1963 	unipro_writel(ufs, 28224, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER1);
1964 	unipro_writel(ufs, 20160, UNIPRO_DME_POWERMODE_REQ_LOCALL2TIMER2);
1965 	unipro_writel(ufs, 12000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER0);
1966 	unipro_writel(ufs, 32000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER1);
1967 	unipro_writel(ufs, 16000, UNIPRO_DME_POWERMODE_REQ_REMOTEL2TIMER2);
1968 
1969 	return 0;
1970 }
1971 
1972 static const struct ufs_hba_variant_ops ufs_hba_exynos_ops = {
1973 	.name				= "exynos_ufs",
1974 	.init				= exynos_ufs_init,
1975 	.hce_enable_notify		= exynos_ufs_hce_enable_notify,
1976 	.link_startup_notify		= exynos_ufs_link_startup_notify,
1977 	.pwr_change_notify		= exynos_ufs_pwr_change_notify,
1978 	.setup_clocks			= exynos_ufs_setup_clocks,
1979 	.setup_xfer_req			= exynos_ufs_specify_nexus_t_xfer_req,
1980 	.setup_task_mgmt		= exynos_ufs_specify_nexus_t_tm_req,
1981 	.hibern8_notify			= exynos_ufs_hibern8_notify,
1982 	.suspend			= exynos_ufs_suspend,
1983 	.resume				= exynos_ufs_resume,
1984 	.fill_crypto_prdt		= exynos_ufs_fmp_fill_prdt,
1985 };
1986 
1987 static struct ufs_hba_variant_ops ufs_hba_exynosauto_vh_ops = {
1988 	.name				= "exynosauto_ufs_vh",
1989 	.init				= exynosauto_ufs_vh_init,
1990 	.link_startup_notify		= exynosauto_ufs_vh_link_startup_notify,
1991 };
1992 
exynos_ufs_probe(struct platform_device * pdev)1993 static int exynos_ufs_probe(struct platform_device *pdev)
1994 {
1995 	int err;
1996 	struct device *dev = &pdev->dev;
1997 	const struct ufs_hba_variant_ops *vops = &ufs_hba_exynos_ops;
1998 	const struct exynos_ufs_drv_data *drv_data =
1999 		device_get_match_data(dev);
2000 
2001 	if (drv_data && drv_data->vops)
2002 		vops = drv_data->vops;
2003 
2004 	err = ufshcd_pltfrm_init(pdev, vops);
2005 	if (err)
2006 		dev_err(dev, "ufshcd_pltfrm_init() failed %d\n", err);
2007 
2008 	return err;
2009 }
2010 
exynos_ufs_remove(struct platform_device * pdev)2011 static void exynos_ufs_remove(struct platform_device *pdev)
2012 {
2013 	struct ufs_hba *hba =  platform_get_drvdata(pdev);
2014 	struct exynos_ufs *ufs = ufshcd_get_variant(hba);
2015 
2016 	ufshcd_pltfrm_remove(pdev);
2017 
2018 	phy_power_off(ufs->phy);
2019 	phy_exit(ufs->phy);
2020 }
2021 
2022 static struct exynos_ufs_uic_attr exynos7_uic_attr = {
2023 	.tx_trailingclks		= 0x10,
2024 	.tx_dif_p_nsec			= 3000000,	/* unit: ns */
2025 	.tx_dif_n_nsec			= 1000000,	/* unit: ns */
2026 	.tx_high_z_cnt_nsec		= 20000,	/* unit: ns */
2027 	.tx_base_unit_nsec		= 100000,	/* unit: ns */
2028 	.tx_gran_unit_nsec		= 4000,		/* unit: ns */
2029 	.tx_sleep_cnt			= 1000,		/* unit: ns */
2030 	.tx_min_activatetime		= 0xa,
2031 	.rx_filler_enable		= 0x2,
2032 	.rx_dif_p_nsec			= 1000000,	/* unit: ns */
2033 	.rx_hibern8_wait_nsec		= 4000000,	/* unit: ns */
2034 	.rx_base_unit_nsec		= 100000,	/* unit: ns */
2035 	.rx_gran_unit_nsec		= 4000,		/* unit: ns */
2036 	.rx_sleep_cnt			= 1280,		/* unit: ns */
2037 	.rx_stall_cnt			= 320,		/* unit: ns */
2038 	.rx_hs_g1_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2039 	.rx_hs_g2_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2040 	.rx_hs_g3_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2041 	.rx_hs_g1_prep_sync_len_cap	= PREP_LEN(0xf),
2042 	.rx_hs_g2_prep_sync_len_cap	= PREP_LEN(0xf),
2043 	.rx_hs_g3_prep_sync_len_cap	= PREP_LEN(0xf),
2044 	.pa_dbg_clk_period_off		= PA_DBG_CLK_PERIOD,
2045 	.pa_dbg_opt_suite1_val		= 0x30103,
2046 	.pa_dbg_opt_suite1_off		= PA_DBG_OPTION_SUITE,
2047 };
2048 
2049 static const struct exynos_ufs_drv_data exynosauto_ufs_drvs = {
2050 	.uic_attr		= &exynos7_uic_attr,
2051 	.quirks			= UFSHCD_QUIRK_PRDT_BYTE_GRAN |
2052 				  UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
2053 				  UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
2054 				  UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING,
2055 	.opts			= EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
2056 				  EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR |
2057 				  EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX,
2058 	.iocc_mask		= UFS_EXYNOSAUTO_SHARABLE,
2059 	.drv_init		= exynosauto_ufs_drv_init,
2060 	.post_hce_enable	= exynosauto_ufs_post_hce_enable,
2061 	.pre_link		= exynosauto_ufs_pre_link,
2062 	.pre_pwr_change		= exynosauto_ufs_pre_pwr_change,
2063 	.post_pwr_change	= exynosauto_ufs_post_pwr_change,
2064 };
2065 
2066 static const struct exynos_ufs_drv_data exynosauto_ufs_vh_drvs = {
2067 	.vops			= &ufs_hba_exynosauto_vh_ops,
2068 	.quirks			= UFSHCD_QUIRK_PRDT_BYTE_GRAN |
2069 				  UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
2070 				  UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
2071 				  UFSHCI_QUIRK_BROKEN_HCE |
2072 				  UFSHCD_QUIRK_BROKEN_UIC_CMD |
2073 				  UFSHCD_QUIRK_SKIP_PH_CONFIGURATION |
2074 				  UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING,
2075 	.opts			= EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX,
2076 };
2077 
2078 static const struct exynos_ufs_drv_data exynos_ufs_drvs = {
2079 	.uic_attr		= &exynos7_uic_attr,
2080 	.quirks			= UFSHCD_QUIRK_PRDT_BYTE_GRAN |
2081 				  UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR |
2082 				  UFSHCI_QUIRK_BROKEN_HCE |
2083 				  UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
2084 				  UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
2085 				  UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL |
2086 				  UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING,
2087 	.opts			= EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL |
2088 				  EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
2089 				  EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX |
2090 				  EXYNOS_UFS_OPT_SKIP_CONNECTION_ESTAB |
2091 				  EXYNOS_UFS_OPT_USE_SW_HIBERN8_TIMER,
2092 	.pre_link		= exynos7_ufs_pre_link,
2093 	.post_link		= exynos7_ufs_post_link,
2094 	.pre_pwr_change		= exynos7_ufs_pre_pwr_change,
2095 	.post_pwr_change	= exynos7_ufs_post_pwr_change,
2096 };
2097 
2098 static struct exynos_ufs_uic_attr gs101_uic_attr = {
2099 	.tx_trailingclks		= 0xff,
2100 	.pa_dbg_opt_suite1_val		= 0x90913C1C,
2101 	.pa_dbg_opt_suite1_off		= PA_GS101_DBG_OPTION_SUITE1,
2102 	.pa_dbg_opt_suite2_val		= 0xE01C115F,
2103 	.pa_dbg_opt_suite2_off		= PA_GS101_DBG_OPTION_SUITE2,
2104 };
2105 
2106 static struct exynos_ufs_uic_attr fsd_uic_attr = {
2107 	.tx_trailingclks		= 0x10,
2108 	.tx_dif_p_nsec			= 3000000,	/* unit: ns */
2109 	.tx_dif_n_nsec			= 1000000,	/* unit: ns */
2110 	.tx_high_z_cnt_nsec		= 20000,	/* unit: ns */
2111 	.tx_base_unit_nsec		= 100000,	/* unit: ns */
2112 	.tx_gran_unit_nsec		= 4000,		/* unit: ns */
2113 	.tx_sleep_cnt			= 1000,		/* unit: ns */
2114 	.tx_min_activatetime		= 0xa,
2115 	.rx_filler_enable		= 0x2,
2116 	.rx_dif_p_nsec			= 1000000,	/* unit: ns */
2117 	.rx_hibern8_wait_nsec		= 4000000,	/* unit: ns */
2118 	.rx_base_unit_nsec		= 100000,	/* unit: ns */
2119 	.rx_gran_unit_nsec		= 4000,		/* unit: ns */
2120 	.rx_sleep_cnt			= 1280,		/* unit: ns */
2121 	.rx_stall_cnt			= 320,		/* unit: ns */
2122 	.rx_hs_g1_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2123 	.rx_hs_g2_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2124 	.rx_hs_g3_sync_len_cap		= SYNC_LEN_COARSE(0xf),
2125 	.rx_hs_g1_prep_sync_len_cap	= PREP_LEN(0xf),
2126 	.rx_hs_g2_prep_sync_len_cap	= PREP_LEN(0xf),
2127 	.rx_hs_g3_prep_sync_len_cap	= PREP_LEN(0xf),
2128 	.pa_dbg_clk_period_off		= PA_DBG_CLK_PERIOD,
2129 	.pa_dbg_opt_suite1_val		= 0x2E820183,
2130 	.pa_dbg_opt_suite1_off		= PA_DBG_OPTION_SUITE,
2131 };
2132 
2133 static const struct exynos_ufs_drv_data fsd_ufs_drvs = {
2134 	.uic_attr               = &fsd_uic_attr,
2135 	.quirks                 = UFSHCD_QUIRK_PRDT_BYTE_GRAN |
2136 				  UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR |
2137 				  UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
2138 				  UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING |
2139 				  UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR,
2140 	.opts                   = EXYNOS_UFS_OPT_HAS_APB_CLK_CTRL |
2141 				  EXYNOS_UFS_OPT_BROKEN_AUTO_CLK_CTRL |
2142 				  EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR |
2143 				  EXYNOS_UFS_OPT_BROKEN_RX_SEL_IDX,
2144 	.pre_link               = fsd_ufs_pre_link,
2145 	.post_link              = fsd_ufs_post_link,
2146 	.pre_pwr_change         = fsd_ufs_pre_pwr_change,
2147 };
2148 
2149 static const struct exynos_ufs_drv_data gs101_ufs_drvs = {
2150 	.uic_attr		= &gs101_uic_attr,
2151 	.quirks			= UFSHCD_QUIRK_PRDT_BYTE_GRAN |
2152 				  UFSHCI_QUIRK_SKIP_RESET_INTR_AGGR |
2153 				  UFSHCI_QUIRK_BROKEN_REQ_LIST_CLR |
2154 				  UFSHCD_QUIRK_BROKEN_OCS_FATAL_ERROR |
2155 				  UFSHCI_QUIRK_SKIP_MANUAL_WB_FLUSH_CTRL |
2156 				  UFSHCD_QUIRK_SKIP_DEF_UNIPRO_TIMEOUT_SETTING,
2157 	.opts			= EXYNOS_UFS_OPT_SKIP_CONFIG_PHY_ATTR |
2158 				  EXYNOS_UFS_OPT_UFSPR_SECURE |
2159 				  EXYNOS_UFS_OPT_TIMER_TICK_SELECT,
2160 	.iocc_mask		= UFS_GS101_SHARABLE,
2161 	.drv_init		= gs101_ufs_drv_init,
2162 	.pre_link		= gs101_ufs_pre_link,
2163 	.post_link		= gs101_ufs_post_link,
2164 	.pre_pwr_change		= gs101_ufs_pre_pwr_change,
2165 };
2166 
2167 static const struct of_device_id exynos_ufs_of_match[] = {
2168 	{ .compatible = "google,gs101-ufs",
2169 	  .data	      = &gs101_ufs_drvs },
2170 	{ .compatible = "samsung,exynos7-ufs",
2171 	  .data	      = &exynos_ufs_drvs },
2172 	{ .compatible = "samsung,exynosautov9-ufs",
2173 	  .data	      = &exynosauto_ufs_drvs },
2174 	{ .compatible = "samsung,exynosautov9-ufs-vh",
2175 	  .data	      = &exynosauto_ufs_vh_drvs },
2176 	{ .compatible = "tesla,fsd-ufs",
2177 	  .data       = &fsd_ufs_drvs },
2178 	{},
2179 };
2180 MODULE_DEVICE_TABLE(of, exynos_ufs_of_match);
2181 
2182 static const struct dev_pm_ops exynos_ufs_pm_ops = {
2183 	SET_SYSTEM_SLEEP_PM_OPS(ufshcd_system_suspend, ufshcd_system_resume)
2184 	SET_RUNTIME_PM_OPS(ufshcd_runtime_suspend, ufshcd_runtime_resume, NULL)
2185 	.prepare	 = ufshcd_suspend_prepare,
2186 	.complete	 = ufshcd_resume_complete,
2187 };
2188 
2189 static struct platform_driver exynos_ufs_pltform = {
2190 	.probe	= exynos_ufs_probe,
2191 	.remove = exynos_ufs_remove,
2192 	.driver	= {
2193 		.name	= "exynos-ufshc",
2194 		.pm	= &exynos_ufs_pm_ops,
2195 		.of_match_table = exynos_ufs_of_match,
2196 	},
2197 };
2198 module_platform_driver(exynos_ufs_pltform);
2199 
2200 MODULE_AUTHOR("Alim Akhtar <[email protected]>");
2201 MODULE_AUTHOR("Seungwon Jeon  <[email protected]>");
2202 MODULE_DESCRIPTION("Exynos UFS HCI Driver");
2203 MODULE_LICENSE("GPL v2");
2204