1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (C) 2015-2016 Marvell International Ltd.
4  */
5 
6 #include <common.h>
7 #include <fdtdec.h>
8 #include <log.h>
9 #include <asm/global_data.h>
10 #include <asm/io.h>
11 #include <asm/arch/cpu.h>
12 #include <asm/arch/soc.h>
13 #include <linux/delay.h>
14 
15 #include "comphy_a3700.h"
16 
17 DECLARE_GLOBAL_DATA_PTR;
18 
19 struct comphy_mux_data a3700_comphy_mux_data[] = {
20 	/* Lane 0 */
21 	{
22 		4,
23 		{
24 			{ COMPHY_TYPE_UNCONNECTED,	0x0 },
25 			{ COMPHY_TYPE_SGMII1,	0x0 },
26 			{ COMPHY_TYPE_USB3_HOST0,	0x1 },
27 			{ COMPHY_TYPE_USB3_DEVICE,	0x1 }
28 		}
29 	},
30 	/* Lane 1 */
31 	{
32 		3,
33 		{
34 			{ COMPHY_TYPE_UNCONNECTED,	0x0},
35 			{ COMPHY_TYPE_SGMII0,	0x0},
36 			{ COMPHY_TYPE_PEX0,	0x1}
37 		}
38 	},
39 	/* Lane 2 */
40 	{
41 		4,
42 		{
43 			{ COMPHY_TYPE_UNCONNECTED,	0x0},
44 			{ COMPHY_TYPE_SATA0,	0x0},
45 			{ COMPHY_TYPE_USB3_HOST0,	0x1},
46 			{ COMPHY_TYPE_USB3_DEVICE,	0x1}
47 		}
48 	},
49 };
50 
51 struct sgmii_phy_init_data_fix {
52 	u16 addr;
53 	u16 value;
54 };
55 
56 /* Changes to 40M1G25 mode data required for running 40M3G125 init mode */
57 static struct sgmii_phy_init_data_fix sgmii_phy_init_fix[] = {
58 	{0x005, 0x07CC}, {0x015, 0x0000}, {0x01B, 0x0000}, {0x01D, 0x0000},
59 	{0x01E, 0x0000}, {0x01F, 0x0000}, {0x020, 0x0000}, {0x021, 0x0030},
60 	{0x026, 0x0888}, {0x04D, 0x0152}, {0x04F, 0xA020}, {0x050, 0x07CC},
61 	{0x053, 0xE9CA}, {0x055, 0xBD97}, {0x071, 0x3015}, {0x076, 0x03AA},
62 	{0x07C, 0x0FDF}, {0x0C2, 0x3030}, {0x0C3, 0x8000}, {0x0E2, 0x5550},
63 	{0x0E3, 0x12A4}, {0x0E4, 0x7D00}, {0x0E6, 0x0C83}, {0x101, 0xFCC0},
64 	{0x104, 0x0C10}
65 };
66 
67 /* 40M1G25 mode init data */
68 static u16 sgmii_phy_init[512] = {
69 	/* 0       1       2       3       4       5       6       7 */
70 	/*-----------------------------------------------------------*/
71 	/* 8       9       A       B       C       D       E       F */
72 	0x3110, 0xFD83, 0x6430, 0x412F, 0x82C0, 0x06FA, 0x4500, 0x6D26,	/* 00 */
73 	0xAFC0, 0x8000, 0xC000, 0x0000, 0x2000, 0x49CC, 0x0BC9, 0x2A52,	/* 08 */
74 	0x0BD2, 0x0CDE, 0x13D2, 0x0CE8, 0x1149, 0x10E0, 0x0000, 0x0000,	/* 10 */
75 	0x0000, 0x0000, 0x0000, 0x0001, 0x0000, 0x4134, 0x0D2D, 0xFFFF,	/* 18 */
76 	0xFFE0, 0x4030, 0x1016, 0x0030, 0x0000, 0x0800, 0x0866, 0x0000,	/* 20 */
77 	0x0000, 0x0000, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF,	/* 28 */
78 	0xFFFF, 0xFFFF, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/* 30 */
79 	0x0000, 0x0000, 0x000F, 0x6A62, 0x1988, 0x3100, 0x3100, 0x3100,	/* 38 */
80 	0x3100, 0xA708, 0x2430, 0x0830, 0x1030, 0x4610, 0xFF00, 0xFF00,	/* 40 */
81 	0x0060, 0x1000, 0x0400, 0x0040, 0x00F0, 0x0155, 0x1100, 0xA02A,	/* 48 */
82 	0x06FA, 0x0080, 0xB008, 0xE3ED, 0x5002, 0xB592, 0x7A80, 0x0001,	/* 50 */
83 	0x020A, 0x8820, 0x6014, 0x8054, 0xACAA, 0xFC88, 0x2A02, 0x45CF,	/* 58 */
84 	0x000F, 0x1817, 0x2860, 0x064F, 0x0000, 0x0204, 0x1800, 0x6000,	/* 60 */
85 	0x810F, 0x4F23, 0x4000, 0x4498, 0x0850, 0x0000, 0x000E, 0x1002,	/* 68 */
86 	0x9D3A, 0x3009, 0xD066, 0x0491, 0x0001, 0x6AB0, 0x0399, 0x3780,	/* 70 */
87 	0x0040, 0x5AC0, 0x4A80, 0x0000, 0x01DF, 0x0000, 0x0007, 0x0000,	/* 78 */
88 	0x2D54, 0x00A1, 0x4000, 0x0100, 0xA20A, 0x0000, 0x0000, 0x0000,	/* 80 */
89 	0x0000, 0x0000, 0x0000, 0x7400, 0x0E81, 0x1000, 0x1242, 0x0210,	/* 88 */
90 	0x80DF, 0x0F1F, 0x2F3F, 0x4F5F, 0x6F7F, 0x0F1F, 0x2F3F, 0x4F5F,	/* 90 */
91 	0x6F7F, 0x4BAD, 0x0000, 0x0000, 0x0800, 0x0000, 0x2400, 0xB651,	/* 98 */
92 	0xC9E0, 0x4247, 0x0A24, 0x0000, 0xAF19, 0x1004, 0x0000, 0x0000,	/* A0 */
93 	0x0000, 0x0013, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/* A8 */
94 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/* B0 */
95 	0x0000, 0x0000, 0x0000, 0x0060, 0x0000, 0x0000, 0x0000, 0x0000,	/* B8 */
96 	0x0000, 0x0000, 0x3010, 0xFA00, 0x0000, 0x0000, 0x0000, 0x0003,	/* C0 */
97 	0x1618, 0x8200, 0x8000, 0x0400, 0x050F, 0x0000, 0x0000, 0x0000,	/* C8 */
98 	0x4C93, 0x0000, 0x1000, 0x1120, 0x0010, 0x1242, 0x1242, 0x1E00,	/* D0 */
99 	0x0000, 0x0000, 0x0000, 0x00F8, 0x0000, 0x0041, 0x0800, 0x0000,	/* D8 */
100 	0x82A0, 0x572E, 0x2490, 0x14A9, 0x4E00, 0x0000, 0x0803, 0x0541,	/* E0 */
101 	0x0C15, 0x0000, 0x0000, 0x0400, 0x2626, 0x0000, 0x0000, 0x4200,	/* E8 */
102 	0x0000, 0xAA55, 0x1020, 0x0000, 0x0000, 0x5010, 0x0000, 0x0000,	/* F0 */
103 	0x0000, 0x0000, 0x5000, 0x0000, 0x0000, 0x0000, 0x02F2, 0x0000,	/* F8 */
104 	0x101F, 0xFDC0, 0x4000, 0x8010, 0x0110, 0x0006, 0x0000, 0x0000,	/*100 */
105 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*108 */
106 	0x04CF, 0x0000, 0x04CF, 0x0000, 0x04CF, 0x0000, 0x04C6, 0x0000,	/*110 */
107 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*118 */
108 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*120 */
109 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*128 */
110 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*130 */
111 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*138 */
112 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*140 */
113 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*148 */
114 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*150 */
115 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*158 */
116 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*160 */
117 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*168 */
118 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*170 */
119 	0x0000, 0x0000, 0x0000, 0x00F0, 0x08A2, 0x3112, 0x0A14, 0x0000,	/*178 */
120 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*180 */
121 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*188 */
122 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*190 */
123 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*198 */
124 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1A0 */
125 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1A8 */
126 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1B0 */
127 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1B8 */
128 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1C0 */
129 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1C8 */
130 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1D0 */
131 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1D8 */
132 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1E0 */
133 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1E8 */
134 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000,	/*1F0 */
135 	0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000	/*1F8 */
136 };
137 
138 /*
139  * comphy_poll_reg
140  *
141  * return: 1 on success, 0 on timeout
142  */
comphy_poll_reg(void * addr,u32 val,u32 mask,u8 op_type)143 static u32 comphy_poll_reg(void *addr, u32 val, u32 mask, u8 op_type)
144 {
145 	u32 rval = 0xDEAD, timeout;
146 
147 	for (timeout = PLL_LOCK_TIMEOUT; timeout > 0; timeout--) {
148 		if (op_type == POLL_16B_REG)
149 			rval = readw(addr);	/* 16 bit */
150 		else
151 			rval = readl(addr) ;	/* 32 bit */
152 
153 		if ((rval & mask) == val)
154 			return 1;
155 
156 		udelay(10000);
157 	}
158 
159 	debug("Time out waiting (%p = %#010x)\n", addr, rval);
160 	return 0;
161 }
162 
163 /*
164  * comphy_pcie_power_up
165  *
166  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
167  */
comphy_pcie_power_up(u32 speed,u32 invert)168 static int comphy_pcie_power_up(u32 speed, u32 invert)
169 {
170 	int ret;
171 
172 	debug_enter();
173 
174 	/*
175 	 * 1. Enable max PLL.
176 	 */
177 	reg_set16(phy_addr(PCIE, LANE_CFG1), bf_use_max_pll_rate, 0);
178 
179 	/*
180 	 * 2. Select 20 bit SERDES interface.
181 	 */
182 	reg_set16(phy_addr(PCIE, GLOB_CLK_SRC_LO), bf_cfg_sel_20b, 0);
183 
184 	/*
185 	 * 3. Force to use reg setting for PCIe mode
186 	 */
187 	reg_set16(phy_addr(PCIE, MISC_REG1), bf_sel_bits_pcie_force, 0);
188 
189 	/*
190 	 * 4. Change RX wait
191 	 */
192 	reg_set16(phy_addr(PCIE, PWR_MGM_TIM1), 0x10C, 0xFFFF);
193 
194 	/*
195 	 * 5. Enable idle sync
196 	 */
197 	reg_set16(phy_addr(PCIE, UNIT_CTRL), 0x60 | rb_idle_sync_en, 0xFFFF);
198 
199 	/*
200 	 * 6. Enable the output of 100M/125M/500M clock
201 	 */
202 	reg_set16(phy_addr(PCIE, MISC_REG0),
203 		  0xA00D | rb_clk500m_en | rb_clk100m_125m_en, 0xFFFF);
204 
205 	/*
206 	 * 7. Enable TX
207 	 */
208 	reg_set(PCIE_REF_CLK_ADDR, 0x1342, 0xFFFFFFFF);
209 
210 	/*
211 	 * 8. Check crystal jumper setting and program the Power and PLL
212 	 *    Control accordingly
213 	 */
214 	if (get_ref_clk() == 40) {
215 		/* 40 MHz */
216 		reg_set16(phy_addr(PCIE, PWR_PLL_CTRL), 0xFC63, 0xFFFF);
217 	} else {
218 		/* 25 MHz */
219 		reg_set16(phy_addr(PCIE, PWR_PLL_CTRL), 0xFC62, 0xFFFF);
220 	}
221 
222 	/*
223 	 * 9. Override Speed_PLL value and use MAC PLL
224 	 */
225 	reg_set16(phy_addr(PCIE, KVCO_CAL_CTRL), 0x0040 | rb_use_max_pll_rate,
226 		  0xFFFF);
227 
228 	/*
229 	 * 10. Check the Polarity invert bit
230 	 */
231 	if (invert & COMPHY_POLARITY_TXD_INVERT)
232 		reg_set16(phy_addr(PCIE, SYNC_PATTERN), phy_txd_inv, 0);
233 
234 	if (invert & COMPHY_POLARITY_RXD_INVERT)
235 		reg_set16(phy_addr(PCIE, SYNC_PATTERN), phy_rxd_inv, 0);
236 
237 	/*
238 	 * 11. Release SW reset
239 	 */
240 	reg_set16(phy_addr(PCIE, GLOB_PHY_CTRL0),
241 		  rb_mode_core_clk_freq_sel | rb_mode_pipe_width_32,
242 		  bf_soft_rst | bf_mode_refdiv);
243 
244 	/* Wait for > 55 us to allow PCLK be enabled */
245 	udelay(PLL_SET_DELAY_US);
246 
247 	/* Assert PCLK enabled */
248 	ret = comphy_poll_reg(phy_addr(PCIE, LANE_STAT1),	/* address */
249 			      rb_txdclk_pclk_en,		/* value */
250 			      rb_txdclk_pclk_en,		/* mask */
251 			      POLL_16B_REG);			/* 16bit */
252 	if (!ret)
253 		printf("Failed to lock PCIe PLL\n");
254 
255 	debug_exit();
256 
257 	/* Return the status of the PLL */
258 	return ret;
259 }
260 
261 /*
262  * reg_set_indirect
263  *
264  * return: void
265  */
reg_set_indirect(u32 reg,u16 data,u16 mask)266 static void reg_set_indirect(u32 reg, u16 data, u16 mask)
267 {
268 	reg_set(rh_vsreg_addr, reg, 0xFFFFFFFF);
269 	reg_set(rh_vsreg_data, data, mask);
270 }
271 
272 /*
273  * comphy_sata_power_up
274  *
275  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
276  */
comphy_sata_power_up(u32 invert)277 static int comphy_sata_power_up(u32 invert)
278 {
279 	int ret;
280 	u32 data = 0;
281 
282 	debug_enter();
283 
284 	/*
285 	 * 0. Check the Polarity invert bits
286 	 */
287 	if (invert & COMPHY_POLARITY_TXD_INVERT)
288 		data |= bs_txd_inv;
289 
290 	if (invert & COMPHY_POLARITY_RXD_INVERT)
291 		data |= bs_rxd_inv;
292 
293 	reg_set_indirect(vphy_sync_pattern_reg, data, bs_txd_inv | bs_rxd_inv);
294 
295 	/*
296 	 * 1. Select 40-bit data width width
297 	 */
298 	reg_set_indirect(vphy_loopback_reg0, 0x800, bs_phyintf_40bit);
299 
300 	/*
301 	 * 2. Select reference clock and PHY mode (SATA)
302 	 */
303 	if (get_ref_clk() == 40) {
304 		/* 40 MHz */
305 		reg_set_indirect(vphy_power_reg0, 0x3, 0x00FF);
306 	} else {
307 		/* 20 MHz */
308 		reg_set_indirect(vphy_power_reg0, 0x1, 0x00FF);
309 	}
310 
311 	/*
312 	 * 3. Use maximum PLL rate (no power save)
313 	 */
314 	reg_set_indirect(vphy_calctl_reg, bs_max_pll_rate, bs_max_pll_rate);
315 
316 	/*
317 	 * 4. Reset reserved bit (??)
318 	 */
319 	reg_set_indirect(vphy_reserve_reg, 0, bs_phyctrl_frm_pin);
320 
321 	/*
322 	 * 5. Set vendor-specific configuration (??)
323 	 */
324 	reg_set(rh_vs0_a, vsata_ctrl_reg, 0xFFFFFFFF);
325 	reg_set(rh_vs0_d, bs_phy_pu_pll, bs_phy_pu_pll);
326 
327 	/* Wait for > 55 us to allow PLL be enabled */
328 	udelay(PLL_SET_DELAY_US);
329 
330 	/* Assert SATA PLL enabled */
331 	reg_set(rh_vsreg_addr, vphy_loopback_reg0, 0xFFFFFFFF);
332 	ret = comphy_poll_reg(rh_vsreg_data,	/* address */
333 			      bs_pll_ready_tx,	/* value */
334 			      bs_pll_ready_tx,	/* mask */
335 			      POLL_32B_REG);	/* 32bit */
336 	if (!ret)
337 		printf("Failed to lock SATA PLL\n");
338 
339 	debug_exit();
340 
341 	return ret;
342 }
343 
344 /*
345  * usb3_reg_set16
346  *
347  * return: void
348  */
usb3_reg_set16(u32 reg,u16 data,u16 mask,u32 lane)349 static void usb3_reg_set16(u32 reg, u16 data, u16 mask, u32 lane)
350 {
351 	/*
352 	 * When Lane 2 PHY is for USB3, access the PHY registers
353 	 * through indirect Address and Data registers INDIR_ACC_PHY_ADDR
354 	 * (RD00E0178h [31:0]) and INDIR_ACC_PHY_DATA (RD00E017Ch [31:0])
355 	 * within the SATA Host Controller registers, Lane 2 base register
356 	 * offset is 0x200
357 	 */
358 
359 	if (lane == 2)
360 		reg_set_indirect(USB3PHY_LANE2_REG_BASE_OFFSET + reg, data,
361 				 mask);
362 	else
363 		reg_set16(phy_addr(USB3, reg), data, mask);
364 }
365 
366 /*
367  * comphy_usb3_power_up
368  *
369  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
370  */
comphy_usb3_power_up(u32 lane,u32 type,u32 speed,u32 invert)371 static int comphy_usb3_power_up(u32 lane, u32 type, u32 speed, u32 invert)
372 {
373 	int ret;
374 
375 	debug_enter();
376 
377 	/*
378 	 * 1. Power up OTG module
379 	 */
380 	reg_set(USB2_PHY_OTG_CTRL_ADDR, rb_pu_otg, 0);
381 
382 	/*
383 	 * 2. Set counter for 100us pulse in USB3 Host and Device
384 	 * restore default burst size limit (Reference Clock 31:24)
385 	 */
386 	reg_set(USB3_CTRPUL_VAL_REG, 0x8 << 24, rb_usb3_ctr_100ns);
387 
388 
389 	/* 0xd005c300 = 0x1001 */
390 	/* set PRD_TXDEEMPH (3.5db de-emph) */
391 	usb3_reg_set16(LANE_CFG0, 0x1, 0xFF, lane);
392 
393 	/*
394 	 * Set BIT0: enable transmitter in high impedance mode
395 	 * Set BIT[3:4]: delay 2 clock cycles for HiZ off latency
396 	 * Set BIT6: Tx detect Rx at HiZ mode
397 	 * Unset BIT15: set to 0 to set USB3 De-emphasize level to -3.5db
398 	 *              together with bit 0 of COMPHY_REG_LANE_CFG0_ADDR
399 	 *              register
400 	 */
401 	usb3_reg_set16(LANE_CFG1,
402 		       tx_det_rx_mode | gen2_tx_data_dly_deft
403 		       | tx_elec_idle_mode_en,
404 		       prd_txdeemph1_mask | tx_det_rx_mode
405 		       | gen2_tx_data_dly_mask | tx_elec_idle_mode_en, lane);
406 
407 	/* 0xd005c310 = 0x93: set Spread Spectrum Clock Enabled */
408 	usb3_reg_set16(LANE_CFG4, bf_spread_spectrum_clock_en, 0x80, lane);
409 
410 	/*
411 	 * set Override Margining Controls From the MAC: Use margining signals
412 	 * from lane configuration
413 	 */
414 	usb3_reg_set16(TEST_MODE_CTRL, rb_mode_margin_override, 0xFFFF, lane);
415 
416 	/* set Lane-to-Lane Bundle Clock Sampling Period = per PCLK cycles */
417 	/* set Mode Clock Source = PCLK is generated from REFCLK */
418 	usb3_reg_set16(GLOB_CLK_SRC_LO, 0x0, 0xFF, lane);
419 
420 	/* set G2 Spread Spectrum Clock Amplitude at 4K */
421 	usb3_reg_set16(GEN2_SETTINGS_2, g2_tx_ssc_amp, 0xF000, lane);
422 
423 	/*
424 	 * unset G3 Spread Spectrum Clock Amplitude & set G3 TX and RX Register
425 	 * Master Current Select
426 	 */
427 	usb3_reg_set16(GEN2_SETTINGS_3, 0x0, 0xFFFF, lane);
428 
429 	/*
430 	 * 3. Check crystal jumper setting and program the Power and PLL
431 	 * Control accordingly
432 	 * 4. Change RX wait
433 	 */
434 	if (get_ref_clk() == 40) {
435 		/* 40 MHz */
436 		usb3_reg_set16(PWR_PLL_CTRL, 0xFCA3, 0xFFFF, lane);
437 		usb3_reg_set16(PWR_MGM_TIM1, 0x10C, 0xFFFF, lane);
438 	} else {
439 		/* 25 MHz */
440 		usb3_reg_set16(PWR_PLL_CTRL, 0xFCA2, 0xFFFF, lane);
441 		usb3_reg_set16(PWR_MGM_TIM1, 0x107, 0xFFFF, lane);
442 	}
443 
444 	/*
445 	 * 5. Enable idle sync
446 	 */
447 	usb3_reg_set16(UNIT_CTRL, 0x60 | rb_idle_sync_en, 0xFFFF, lane);
448 
449 	/*
450 	 * 6. Enable the output of 500M clock
451 	 */
452 	usb3_reg_set16(MISC_REG0, 0xA00D | rb_clk500m_en, 0xFFFF, lane);
453 
454 	/*
455 	 * 7. Set 20-bit data width
456 	 */
457 	usb3_reg_set16(DIG_LB_EN, 0x0400, 0xFFFF, lane);
458 
459 	/*
460 	 * 8. Override Speed_PLL value and use MAC PLL
461 	 */
462 	usb3_reg_set16(KVCO_CAL_CTRL, 0x0040 | rb_use_max_pll_rate, 0xFFFF,
463 		       lane);
464 
465 	/*
466 	 * 9. Check the Polarity invert bit
467 	 */
468 	if (invert & COMPHY_POLARITY_TXD_INVERT)
469 		usb3_reg_set16(SYNC_PATTERN, phy_txd_inv, 0, lane);
470 
471 	if (invert & COMPHY_POLARITY_RXD_INVERT)
472 		usb3_reg_set16(SYNC_PATTERN, phy_rxd_inv, 0, lane);
473 
474 	/*
475 	 * 10. Set max speed generation to USB3.0 5Gbps
476 	 */
477 	usb3_reg_set16(SYNC_MASK_GEN, 0x0400, 0x0C00, lane);
478 
479 	/*
480 	 * 11. Set capacitor value for FFE gain peaking to 0xF
481 	 */
482 	usb3_reg_set16(GEN3_SETTINGS_3, 0xF, 0xF, lane);
483 
484 	/*
485 	 * 12. Release SW reset
486 	 */
487 	usb3_reg_set16(GLOB_PHY_CTRL0,
488 		       rb_mode_core_clk_freq_sel | rb_mode_pipe_width_32
489 		       | 0x20, 0xFFFF, lane);
490 
491 	/* Wait for > 55 us to allow PCLK be enabled */
492 	udelay(PLL_SET_DELAY_US);
493 
494 	/* Assert PCLK enabled */
495 	if (lane == 2) {
496 		reg_set(rh_vsreg_addr,
497 			LANE_STAT1 + USB3PHY_LANE2_REG_BASE_OFFSET,
498 			0xFFFFFFFF);
499 		ret = comphy_poll_reg(rh_vsreg_data,		/* address */
500 				      rb_txdclk_pclk_en,	/* value */
501 				      rb_txdclk_pclk_en,	/* mask */
502 				      POLL_32B_REG);		/* 32bit */
503 	} else {
504 		ret = comphy_poll_reg(phy_addr(USB3, LANE_STAT1), /* address */
505 				      rb_txdclk_pclk_en,	  /* value */
506 				      rb_txdclk_pclk_en,	  /* mask */
507 				      POLL_16B_REG);		  /* 16bit */
508 	}
509 	if (!ret)
510 		printf("Failed to lock USB3 PLL\n");
511 
512 	/*
513 	 * Set Soft ID for Host mode (Device mode works with Hard ID
514 	 * detection)
515 	 */
516 	if (type == COMPHY_TYPE_USB3_HOST0) {
517 		/*
518 		 * set   BIT0: set ID_MODE of Host/Device = "Soft ID" (BIT1)
519 		 * clear BIT1: set SOFT_ID = Host
520 		 * set   BIT4: set INT_MODE = ID. Interrupt Mode: enable
521 		 *             interrupt by ID instead of using both interrupts
522 		 *             of HOST and Device ORed simultaneously
523 		 *             INT_MODE=ID in order to avoid unexpected
524 		 *             behaviour or both interrupts together
525 		 */
526 		reg_set(USB32_CTRL_BASE,
527 			usb32_ctrl_id_mode | usb32_ctrl_int_mode,
528 			usb32_ctrl_id_mode | usb32_ctrl_soft_id |
529 			usb32_ctrl_int_mode);
530 	}
531 
532 	debug_exit();
533 
534 	return ret;
535 }
536 
537 /*
538  * comphy_usb2_power_up
539  *
540  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
541  */
comphy_usb2_power_up(u8 usb32)542 static int comphy_usb2_power_up(u8 usb32)
543 {
544 	int ret;
545 
546 	debug_enter();
547 
548 	if (usb32 != 0 && usb32 != 1) {
549 		printf("invalid usb32 value: (%d), should be either 0 or 1\n",
550 		       usb32);
551 		debug_exit();
552 		return 0;
553 	}
554 
555 	/*
556 	 * 0. Setup PLL. 40MHz clock uses defaults.
557 	 *    See "PLL Settings for Typical REFCLK" table
558 	 */
559 	if (get_ref_clk() == 25) {
560 		reg_set(USB2_PHY_BASE(usb32), 5 | (96 << 16),
561 			0x3F | (0xFF << 16) | (0x3 << 28));
562 	}
563 
564 	/*
565 	 * 1. PHY pull up and disable USB2 suspend
566 	 */
567 	reg_set(USB2_PHY_CTRL_ADDR(usb32),
568 		RB_USB2PHY_SUSPM(usb32) | RB_USB2PHY_PU(usb32), 0);
569 
570 	if (usb32 != 0) {
571 		/*
572 		 * 2. Power up OTG module
573 		 */
574 		reg_set(USB2_PHY_OTG_CTRL_ADDR, rb_pu_otg, 0);
575 
576 		/*
577 		 * 3. Configure PHY charger detection
578 		 */
579 		reg_set(USB2_PHY_CHRGR_DET_ADDR, 0,
580 			rb_cdp_en | rb_dcp_en | rb_pd_en | rb_cdp_dm_auto |
581 			rb_enswitch_dp | rb_enswitch_dm | rb_pu_chrg_dtc);
582 	}
583 
584 	/* Assert PLL calibration done */
585 	ret = comphy_poll_reg(USB2_PHY_CAL_CTRL_ADDR(usb32),
586 			      rb_usb2phy_pllcal_done,	/* value */
587 			      rb_usb2phy_pllcal_done,	/* mask */
588 			      POLL_32B_REG);		/* 32bit */
589 	if (!ret)
590 		printf("Failed to end USB2 PLL calibration\n");
591 
592 	/* Assert impedance calibration done */
593 	ret = comphy_poll_reg(USB2_PHY_CAL_CTRL_ADDR(usb32),
594 			      rb_usb2phy_impcal_done,	/* value */
595 			      rb_usb2phy_impcal_done,	/* mask */
596 			      POLL_32B_REG);		/* 32bit */
597 	if (!ret)
598 		printf("Failed to end USB2 impedance calibration\n");
599 
600 	/* Assert squetch calibration done */
601 	ret = comphy_poll_reg(USB2_PHY_RX_CHAN_CTRL1_ADDR(usb32),
602 			      rb_usb2phy_sqcal_done,	/* value */
603 			      rb_usb2phy_sqcal_done,	/* mask */
604 			      POLL_32B_REG);		/* 32bit */
605 	if (!ret)
606 		printf("Failed to end USB2 unknown calibration\n");
607 
608 	/* Assert PLL is ready */
609 	ret = comphy_poll_reg(USB2_PHY_PLL_CTRL0_ADDR(usb32),
610 			      rb_usb2phy_pll_ready,		/* value */
611 			      rb_usb2phy_pll_ready,		/* mask */
612 			      POLL_32B_REG);		/* 32bit */
613 
614 	if (!ret)
615 		printf("Failed to lock USB2 PLL\n");
616 
617 	debug_exit();
618 
619 	return ret;
620 }
621 
622 /*
623  * comphy_emmc_power_up
624  *
625  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
626  */
comphy_emmc_power_up(void)627 static int comphy_emmc_power_up(void)
628 {
629 	debug_enter();
630 
631 	/*
632 	 * 1. Bus power ON, Bus voltage 1.8V
633 	 */
634 	reg_set(SDIO_HOST_CTRL1_ADDR, 0xB00, 0xF00);
635 
636 	/*
637 	 * 2. Set FIFO parameters
638 	 */
639 	reg_set(SDIO_SDHC_FIFO_ADDR, 0x315, 0xFFFFFFFF);
640 
641 	/*
642 	 * 3. Set Capabilities 1_2
643 	 */
644 	reg_set(SDIO_CAP_12_ADDR, 0x25FAC8B2, 0xFFFFFFFF);
645 
646 	/*
647 	 * 4. Set Endian
648 	 */
649 	reg_set(SDIO_ENDIAN_ADDR, 0x00c00000, 0);
650 
651 	/*
652 	 * 4. Init PHY
653 	 */
654 	reg_set(SDIO_PHY_TIMING_ADDR, 0x80000000, 0x80000000);
655 	reg_set(SDIO_PHY_PAD_CTRL0_ADDR, 0x50000000, 0xF0000000);
656 
657 	/*
658 	 * 5. DLL reset
659 	 */
660 	reg_set(SDIO_DLL_RST_ADDR, 0xFFFEFFFF, 0);
661 	reg_set(SDIO_DLL_RST_ADDR, 0x00010000, 0);
662 
663 	debug_exit();
664 
665 	return 1;
666 }
667 
668 /*
669  * comphy_sgmii_power_up
670  *
671  * return:
672  */
comphy_sgmii_phy_init(u32 lane,u32 speed)673 static void comphy_sgmii_phy_init(u32 lane, u32 speed)
674 {
675 	const int fix_arr_sz = ARRAY_SIZE(sgmii_phy_init_fix);
676 	int addr, fix_idx;
677 	u16 val;
678 
679 	fix_idx = 0;
680 	for (addr = 0; addr < 512; addr++) {
681 		/*
682 		 * All PHY register values are defined in full for 3.125Gbps
683 		 * SERDES speed. The values required for 1.25 Gbps are almost
684 		 * the same and only few registers should be "fixed" in
685 		 * comparison to 3.125 Gbps values. These register values are
686 		 * stored in "sgmii_phy_init_fix" array.
687 		 */
688 		if (speed != COMPHY_SPEED_1_25G &&
689 		    sgmii_phy_init_fix[fix_idx].addr == addr) {
690 			/* Use new value */
691 			val = sgmii_phy_init_fix[fix_idx].value;
692 			if (fix_idx < fix_arr_sz)
693 				fix_idx++;
694 		} else {
695 			val = sgmii_phy_init[addr];
696 		}
697 
698 		reg_set16(sgmiiphy_addr(lane, addr), val, 0xFFFF);
699 	}
700 }
701 
702 /*
703  * comphy_sgmii_power_up
704  *
705  * return: 1 if PLL locked (OK), 0 otherwise (FAIL)
706  */
comphy_sgmii_power_up(u32 lane,u32 speed,u32 invert)707 static int comphy_sgmii_power_up(u32 lane, u32 speed, u32 invert)
708 {
709 	int ret;
710 	u32 saved_selector;
711 
712 	debug_enter();
713 
714 	/*
715 	 * 1. Configure PHY to SATA/SAS mode by setting pin PIN_PIPE_SEL=0
716 	 */
717 	saved_selector = readl(COMPHY_SEL_ADDR);
718 	reg_set(COMPHY_SEL_ADDR, 0, 0xFFFFFFFF);
719 
720 	/*
721 	 * 2. Reset PHY by setting PHY input port PIN_RESET=1.
722 	 * 3. Set PHY input port PIN_TX_IDLE=1, PIN_PU_IVREF=1 to keep
723 	 *    PHY TXP/TXN output to idle state during PHY initialization
724 	 * 4. Set PHY input port PIN_PU_PLL=0, PIN_PU_RX=0, PIN_PU_TX=0.
725 	 */
726 	reg_set(COMPHY_PHY_CFG1_ADDR(lane),
727 		rb_pin_reset_comphy | rb_pin_tx_idle | rb_pin_pu_iveref,
728 		rb_pin_reset_core | rb_pin_pu_pll |
729 		rb_pin_pu_rx | rb_pin_pu_tx);
730 
731 	/*
732 	 * 5. Release reset to the PHY by setting PIN_RESET=0.
733 	 */
734 	reg_set(COMPHY_PHY_CFG1_ADDR(lane), 0, rb_pin_reset_comphy);
735 
736 	/*
737 	 * 7. Set PIN_PHY_GEN_TX[3:0] and PIN_PHY_GEN_RX[3:0] to decide
738 	 *    COMPHY bit rate
739 	 */
740 	if (speed == COMPHY_SPEED_3_125G) { /* 3.125 GHz */
741 		reg_set(COMPHY_PHY_CFG1_ADDR(lane),
742 			(0x8 << rf_gen_rx_sel_shift) |
743 			(0x8 << rf_gen_tx_sel_shift),
744 			rf_gen_rx_select | rf_gen_tx_select);
745 
746 	} else if (speed == COMPHY_SPEED_1_25G) { /* 1.25 GHz */
747 		reg_set(COMPHY_PHY_CFG1_ADDR(lane),
748 			(0x6 << rf_gen_rx_sel_shift) |
749 			(0x6 << rf_gen_tx_sel_shift),
750 			rf_gen_rx_select | rf_gen_tx_select);
751 	} else {
752 		printf("Unsupported COMPHY speed!\n");
753 		return 0;
754 	}
755 
756 	/*
757 	 * 8. Wait 1mS for bandgap and reference clocks to stabilize;
758 	 *    then start SW programming.
759 	 */
760 	mdelay(10);
761 
762 	/* 9. Program COMPHY register PHY_MODE */
763 	reg_set16(sgmiiphy_addr(lane, PWR_PLL_CTRL),
764 		  PHY_MODE_SGMII << rf_phy_mode_shift, rf_phy_mode_mask);
765 
766 	/*
767 	 * 10. Set COMPHY register REFCLK_SEL to select the correct REFCLK
768 	 *     source
769 	 */
770 	reg_set16(sgmiiphy_addr(lane, MISC_REG0), 0, rb_ref_clk_sel);
771 
772 	/*
773 	 * 11. Set correct reference clock frequency in COMPHY register
774 	 *     REF_FREF_SEL.
775 	 */
776 	if (get_ref_clk() == 40) {
777 		reg_set16(sgmiiphy_addr(lane, PWR_PLL_CTRL),
778 			  0x4 << rf_ref_freq_sel_shift, rf_ref_freq_sel_mask);
779 	} else {
780 		/* 25MHz */
781 		reg_set16(sgmiiphy_addr(lane, PWR_PLL_CTRL),
782 			  0x1 << rf_ref_freq_sel_shift, rf_ref_freq_sel_mask);
783 	}
784 
785 	/* 12. Program COMPHY register PHY_GEN_MAX[1:0] */
786 	/*
787 	 * This step is mentioned in the flow received from verification team.
788 	 * However the PHY_GEN_MAX value is only meaningful for other
789 	 * interfaces (not SGMII). For instance, it selects SATA speed
790 	 * 1.5/3/6 Gbps or PCIe speed  2.5/5 Gbps
791 	 */
792 
793 	/*
794 	 * 13. Program COMPHY register SEL_BITS to set correct parallel data
795 	 *     bus width
796 	 */
797 	/* 10bit */
798 	reg_set16(sgmiiphy_addr(lane, DIG_LB_EN), 0, rf_data_width_mask);
799 
800 	/*
801 	 * 14. As long as DFE function needs to be enabled in any mode,
802 	 *     COMPHY register DFE_UPDATE_EN[5:0] shall be programmed to 0x3F
803 	 *     for real chip during COMPHY power on.
804 	 */
805 	/*
806 	 * The step 14 exists (and empty) in the original initialization flow
807 	 * obtained from the verification team. According to the functional
808 	 * specification DFE_UPDATE_EN already has the default value 0x3F
809 	 */
810 
811 	/*
812 	 * 15. Program COMPHY GEN registers.
813 	 *     These registers should be programmed based on the lab testing
814 	 *     result to achieve optimal performance. Please contact the CEA
815 	 *     group to get the related GEN table during real chip bring-up.
816 	 *     We only requred to run though the entire registers programming
817 	 *     flow defined by "comphy_sgmii_phy_init" when the REF clock is
818 	 *     40 MHz. For REF clock 25 MHz the default values stored in PHY
819 	 *     registers are OK.
820 	 */
821 	debug("Running C-DPI phy init %s mode\n",
822 	      speed == COMPHY_SPEED_3_125G ? "2G5" : "1G");
823 	if (get_ref_clk() == 40)
824 		comphy_sgmii_phy_init(lane, speed);
825 
826 	/*
827 	 * 16. [Simulation Only] should not be used for real chip.
828 	 *     By pass power up calibration by programming EXT_FORCE_CAL_DONE
829 	 *     (R02h[9]) to 1 to shorten COMPHY simulation time.
830 	 */
831 	/*
832 	 * 17. [Simulation Only: should not be used for real chip]
833 	 *     Program COMPHY register FAST_DFE_TIMER_EN=1 to shorten RX
834 	 *     training simulation time.
835 	 */
836 
837 	/*
838 	 * 18. Check the PHY Polarity invert bit
839 	 */
840 	if (invert & COMPHY_POLARITY_TXD_INVERT)
841 		reg_set16(sgmiiphy_addr(lane, SYNC_PATTERN), phy_txd_inv, 0);
842 
843 	if (invert & COMPHY_POLARITY_RXD_INVERT)
844 		reg_set16(sgmiiphy_addr(lane, SYNC_PATTERN), phy_rxd_inv, 0);
845 
846 	/*
847 	 * 19. Set PHY input ports PIN_PU_PLL, PIN_PU_TX and PIN_PU_RX to 1
848 	 *     to start PHY power up sequence. All the PHY register
849 	 *     programming should be done before PIN_PU_PLL=1. There should be
850 	 *     no register programming for normal PHY operation from this point.
851 	 */
852 	reg_set(COMPHY_PHY_CFG1_ADDR(lane),
853 		rb_pin_pu_pll | rb_pin_pu_rx | rb_pin_pu_tx,
854 		rb_pin_pu_pll | rb_pin_pu_rx | rb_pin_pu_tx);
855 
856 	/*
857 	 * 20. Wait for PHY power up sequence to finish by checking output ports
858 	 *     PIN_PLL_READY_TX=1 and PIN_PLL_READY_RX=1.
859 	 */
860 	ret = comphy_poll_reg(COMPHY_PHY_STAT1_ADDR(lane),	/* address */
861 			      rb_pll_ready_tx | rb_pll_ready_rx, /* value */
862 			      rb_pll_ready_tx | rb_pll_ready_rx, /* mask */
863 			      POLL_32B_REG);			/* 32bit */
864 	if (!ret)
865 		printf("Failed to lock PLL for SGMII PHY %d\n", lane);
866 
867 	/*
868 	 * 21. Set COMPHY input port PIN_TX_IDLE=0
869 	 */
870 	reg_set(COMPHY_PHY_CFG1_ADDR(lane), 0x0, rb_pin_tx_idle);
871 
872 	/*
873 	 * 22. After valid data appear on PIN_RXDATA bus, set PIN_RX_INIT=1.
874 	 *     to start RX initialization. PIN_RX_INIT_DONE will be cleared to
875 	 *     0 by the PHY. After RX initialization is done, PIN_RX_INIT_DONE
876 	 *     will be set to 1 by COMPHY. Set PIN_RX_INIT=0 after
877 	 *     PIN_RX_INIT_DONE= 1.
878 	 *     Please refer to RX initialization part for details.
879 	 */
880 	reg_set(COMPHY_PHY_CFG1_ADDR(lane), rb_phy_rx_init, 0x0);
881 
882 	ret = comphy_poll_reg(COMPHY_PHY_STAT1_ADDR(lane), /* address */
883 			      rb_rx_init_done,			/* value */
884 			      rb_rx_init_done,			/* mask */
885 			      POLL_32B_REG);			/* 32bit */
886 	if (!ret)
887 		printf("Failed to init RX of SGMII PHY %d\n", lane);
888 
889 	/*
890 	 * Restore saved selector.
891 	 */
892 	reg_set(COMPHY_SEL_ADDR, saved_selector, 0xFFFFFFFF);
893 
894 	debug_exit();
895 
896 	return ret;
897 }
898 
comphy_dedicated_phys_init(void)899 void comphy_dedicated_phys_init(void)
900 {
901 	int node, usb32, ret = 1;
902 	const void *blob = gd->fdt_blob;
903 
904 	debug_enter();
905 
906 	for (usb32 = 0; usb32 <= 1; usb32++) {
907 		/*
908 		 * There are 2 UTMI PHYs in this SOC.
909 		 * One is independendent and one is paired with USB3 port (OTG)
910 		 */
911 		if (usb32 == 0) {
912 			node = fdt_node_offset_by_compatible(
913 				blob, -1, "marvell,armada3700-ehci");
914 		} else {
915 			node = fdt_node_offset_by_compatible(
916 				blob, -1, "marvell,armada3700-xhci");
917 		}
918 
919 		if (node > 0) {
920 			if (fdtdec_get_is_enabled(blob, node)) {
921 				ret = comphy_usb2_power_up(usb32);
922 				if (!ret)
923 					printf("Failed to initialize UTMI PHY\n");
924 				else
925 					debug("UTMI PHY init succeed\n");
926 			} else {
927 				debug("USB%d node is disabled\n",
928 				      usb32 == 0 ? 2 : 3);
929 			}
930 		} else {
931 			debug("No USB%d node in DT\n", usb32 == 0 ? 2 : 3);
932 		}
933 	}
934 
935 	node = fdt_node_offset_by_compatible(blob, -1,
936 					     "marvell,armada-8k-sdhci");
937 	if (node <= 0) {
938 		node = fdt_node_offset_by_compatible(
939 			blob, -1, "marvell,armada-3700-sdhci");
940 	}
941 
942 	if (node > 0) {
943 		if (fdtdec_get_is_enabled(blob, node)) {
944 			ret = comphy_emmc_power_up();
945 			if (!ret)
946 				printf("Failed to initialize SDIO/eMMC PHY\n");
947 			else
948 				debug("SDIO/eMMC PHY init succeed\n");
949 		} else {
950 			debug("SDIO/eMMC node is disabled\n");
951 		}
952 	}  else {
953 		debug("No SDIO/eMMC node in DT\n");
954 	}
955 
956 	debug_exit();
957 }
958 
comphy_a3700_init(struct chip_serdes_phy_config * chip_cfg,struct comphy_map * serdes_map)959 int comphy_a3700_init(struct chip_serdes_phy_config *chip_cfg,
960 		      struct comphy_map *serdes_map)
961 {
962 	struct comphy_map *comphy_map;
963 	u32 comphy_max_count = chip_cfg->comphy_lanes_count;
964 	u32 lane, ret = 0;
965 
966 	debug_enter();
967 
968 	/* Initialize PHY mux */
969 	chip_cfg->mux_data = a3700_comphy_mux_data;
970 	comphy_mux_init(chip_cfg, serdes_map, COMPHY_SEL_ADDR);
971 
972 	for (lane = 0, comphy_map = serdes_map; lane < comphy_max_count;
973 	     lane++, comphy_map++) {
974 		debug("Initialize serdes number %d\n", lane);
975 		debug("Serdes type = 0x%x invert=%d\n",
976 		      comphy_map->type, comphy_map->invert);
977 
978 		switch (comphy_map->type) {
979 		case COMPHY_TYPE_UNCONNECTED:
980 			continue;
981 			break;
982 
983 		case COMPHY_TYPE_PEX0:
984 			ret = comphy_pcie_power_up(comphy_map->speed,
985 						   comphy_map->invert);
986 			break;
987 
988 		case COMPHY_TYPE_USB3_HOST0:
989 		case COMPHY_TYPE_USB3_DEVICE:
990 			ret = comphy_usb3_power_up(lane,
991 						   comphy_map->type,
992 						   comphy_map->speed,
993 						   comphy_map->invert);
994 			break;
995 
996 		case COMPHY_TYPE_SGMII0:
997 		case COMPHY_TYPE_SGMII1:
998 			ret = comphy_sgmii_power_up(lane, comphy_map->speed,
999 						    comphy_map->invert);
1000 			break;
1001 
1002 		case COMPHY_TYPE_SATA0:
1003 			ret = comphy_sata_power_up(comphy_map->invert);
1004 			break;
1005 
1006 		default:
1007 			debug("Unknown SerDes type, skip initialize SerDes %d\n",
1008 			      lane);
1009 			ret = 1;
1010 			break;
1011 		}
1012 		if (!ret)
1013 			printf("PLL is not locked - Failed to initialize lane %d\n",
1014 			       lane);
1015 	}
1016 
1017 	debug_exit();
1018 	return ret;
1019 }
1020