arm: am437x: Enable hardware leveling for EMIF

Switch to using hardware leveling for certain parameters on the EMIF
rather than using precalculated values.  Doing this also means we have a
common place now between am437x and am335x for setting
emif_sdram_ref_ctrl with a value for the correct delay length.

Tested-by: Felipe Balbi <balbi@ti.com>
Tested-by: Tom Rini <trini@ti.com>
Signed-off-by: James Doublesin <doublesin@ti.com>
Signed-off-by: Felipe Balbi <balbi@ti.com>

7 files changed, 138 insertions, 207 deletions

diff --git a/arch/arm/cpu/armv7/am33xx/ddr.c b/arch/arm/cpu/armv7/am33xx/ddr.c
index fc66872a31..85cceae152 100644
--- a/arch/arm/cpu/armv7/am33xx/ddr.c
+++ b/arch/arm/cpu/armv7/am33xx/ddr.c
@@ -76,13 +76,13 @@ static void configure_mr(int nr, u32 cs)
 }
 
 /*
- * Configure EMIF4D5 registers and MR registers
+ * Configure EMIF4D5 registers and MR registers For details about these magic
+ * values please see the EMIF registers section of the TRM.
  */
 void config_sdram_emif4d5(const struct emif_regs *regs, int nr)
 {
 	writel(0xA0, &emif_reg[nr]->emif_pwr_mgmt_ctrl);
 	writel(0xA0, &emif_reg[nr]->emif_pwr_mgmt_ctrl_shdw);
-	writel(0x1, &emif_reg[nr]->emif_iodft_tlgc);
 	writel(regs->zq_config, &emif_reg[nr]->emif_zq_config);
 
 	writel(regs->temp_alert_config, &emif_reg[nr]->emif_temp_alert_config);
@@ -106,10 +106,45 @@ void config_sdram_emif4d5(const struct emif_regs *regs, int nr)
 		writel(regs->emif_cos_config, &emif_reg[nr]->emif_cos_config);
 	}
 
-	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl);
-	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl_shdw);
+	/*
+	 * Sequence to ensure that the PHY is in a known state prior to
+	 * startting hardware leveling.  Also acts as to latch some state from
+	 * the EMIF into the PHY.
+	 */
+	writel(0x2011, &emif_reg[nr]->emif_iodft_tlgc);
+	writel(0x2411, &emif_reg[nr]->emif_iodft_tlgc);
+	writel(0x2011, &emif_reg[nr]->emif_iodft_tlgc);
+
+	clrbits_le32(&emif_reg[nr]->emif_sdram_ref_ctrl,
+			EMIF_REG_INITREF_DIS_MASK);
+
 	writel(regs->sdram_config, &emif_reg[nr]->emif_sdram_config);
 	writel(regs->sdram_config, &cstat->secure_emif_sdram_config);
+	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl);
+	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl_shdw);
+
+	/* Perform hardware leveling. */
+	udelay(1000);
+	writel(readl(&emif_reg[nr]->emif_ddr_ext_phy_ctrl_36) |
+	       0x100, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_36);
+	writel(readl(&emif_reg[nr]->emif_ddr_ext_phy_ctrl_36_shdw) |
+	       0x100, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_36_shdw);
+
+	writel(0x80000000, &emif_reg[nr]->emif_rd_wr_lvl_rmp_ctl);
+
+	/* Enable read leveling */
+	writel(0x80000000, &emif_reg[nr]->emif_rd_wr_lvl_ctl);
+
+	/*
+	 * Enable full read and write leveling.  Wait for read and write
+	 * leveling bit to clear RDWRLVLFULL_START bit 31
+	 */
+	while((readl(&emif_reg[nr]->emif_rd_wr_lvl_ctl) & 0x80000000) != 0)
+		;
+
+	/* Check the timeout register to see if leveling is complete */
+	if((readl(&emif_reg[nr]->emif_status) & 0x70) != 0)
+		puts("DDR3 H/W leveling incomplete with errors\n");
 
 	if (emif_sdram_type() == EMIF_SDRAM_TYPE_LPDDR2) {
 		configure_mr(nr, 0);
@@ -123,21 +158,15 @@ void config_sdram_emif4d5(const struct emif_regs *regs, int nr)
 void config_sdram(const struct emif_regs *regs, int nr)
 {
 	if (regs->zq_config) {
-		/*
-		 * A value of 0x2800 for the REF CTRL will give us
-		 * about 570us for a delay, which will be long enough
-		 * to configure things.
-		 */
-		writel(0x2800, &emif_reg[nr]->emif_sdram_ref_ctrl);
 		writel(regs->zq_config, &emif_reg[nr]->emif_zq_config);
 		writel(regs->sdram_config, &cstat->secure_emif_sdram_config);
 		writel(regs->sdram_config, &emif_reg[nr]->emif_sdram_config);
 		writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl);
 		writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl_shdw);
 	}
+	writel(regs->sdram_config, &emif_reg[nr]->emif_sdram_config);
 	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl);
 	writel(regs->ref_ctrl, &emif_reg[nr]->emif_sdram_ref_ctrl_shdw);
-	writel(regs->sdram_config, &emif_reg[nr]->emif_sdram_config);
 }
 
 /**
@@ -153,46 +182,55 @@ void set_sdram_timings(const struct emif_regs *regs, int nr)
 	writel(regs->sdram_tim3, &emif_reg[nr]->emif_sdram_tim_3_shdw);
 }
 
-void __weak emif_get_ext_phy_ctrl_const_regs(const u32 **regs, u32 *size)
-{
-}
-
 /*
- * Configure EXT PHY registers
+ * Configure EXT PHY registers for hardware leveling
  */
 static void ext_phy_settings(const struct emif_regs *regs, int nr)
 {
-	u32 *ext_phy_ctrl_base = 0;
-	u32 *emif_ext_phy_ctrl_base = 0;
-	const u32 *ext_phy_ctrl_const_regs;
-	u32 i = 0;
-	u32 size;
-
-	ext_phy_ctrl_base = (u32 *)&(regs->emif_ddr_ext_phy_ctrl_1);
-	emif_ext_phy_ctrl_base =
-			(u32 *)&(emif_reg[nr]->emif_ddr_ext_phy_ctrl_1);
-
-	/* Configure external phy control timing registers */
-	for (i = 0; i < EMIF_EXT_PHY_CTRL_TIMING_REG; i++) {
-		writel(*ext_phy_ctrl_base, emif_ext_phy_ctrl_base++);
-		/* Update shadow registers */
-		writel(*ext_phy_ctrl_base++, emif_ext_phy_ctrl_base++);
-	}
-
 	/*
-	 * external phy 6-24 registers do not change with
-	 * ddr frequency
+	 * Enable hardware leveling on the EMIF.  For details about these
+	 * magic values please see the EMIF registers section of the TRM.
 	 */
-	emif_get_ext_phy_ctrl_const_regs(&ext_phy_ctrl_const_regs, &size);
-
-	if (!size)
-		return;
+	writel(0x08020080, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_1);
+	writel(0x08020080, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_1_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_22);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_22_shdw);
+	writel(0x00600020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_23);
+	writel(0x00600020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_23_shdw);
+	writel(0x40010080, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_24);
+	writel(0x40010080, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_24_shdw);
+	writel(0x08102040, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_25);
+	writel(0x08102040, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_25_shdw);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_26);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_26_shdw);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_27);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_27_shdw);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_28);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_28_shdw);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_29);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_29_shdw);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_30);
+	writel(0x00200020, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_30_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_31);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_31_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_32);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_32_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_33);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_33_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_34);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_34_shdw);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_35);
+	writel(0x00000000, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_35_shdw);
+	writel(0x000000FF, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_36);
+	writel(0x000000FF, &emif_reg[nr]->emif_ddr_ext_phy_ctrl_36_shdw);
 
-	for (i = 0; i < size; i++) {
-		writel(ext_phy_ctrl_const_regs[i], emif_ext_phy_ctrl_base++);
-		/* Update shadow registers */
-		writel(ext_phy_ctrl_const_regs[i], emif_ext_phy_ctrl_base++);
-	}
+	/*
+	 * Sequence to ensure that the PHY is again in a known state after
+	 * hardware leveling.
+	 */
+	writel(0x2011, &emif_reg[nr]->emif_iodft_tlgc);
+	writel(0x2411, &emif_reg[nr]->emif_iodft_tlgc);
+	writel(0x2011, &emif_reg[nr]->emif_iodft_tlgc);
 }
 
 /**
@@ -201,11 +239,17 @@ static void ext_phy_settings(const struct emif_regs *regs, int nr)
 void config_ddr_phy(const struct emif_regs *regs, int nr)
 {
 	/*
-	 * disable initialization and refreshes for now until we
+	 * Disable initialization and refreshes for now until we
 	 * finish programming EMIF regs.
+	 * Also set time between rising edge of DDR_RESET to rising
+	 * edge of DDR_CKE to > 500us per memory spec.
 	 */
+#ifndef CONFIG_AM43XX
 	setbits_le32(&emif_reg[nr]->emif_sdram_ref_ctrl,
 		     EMIF_REG_INITREF_DIS_MASK);
+#endif
+	if (regs->zq_config)
+		writel(0x80003100, &emif_reg[nr]->emif_sdram_ref_ctrl);
 
 	writel(regs->emif_ddr_phy_ctlr_1,
 		&emif_reg[nr]->emif_ddr_phy_ctrl_1);
diff --git a/arch/arm/cpu/armv7/am33xx/emif4.c b/arch/arm/cpu/armv7/am33xx/emif4.c
index 8b7527c5b4..9cf816c89a 100644
--- a/arch/arm/cpu/armv7/am33xx/emif4.c
+++ b/arch/arm/cpu/armv7/am33xx/emif4.c
@@ -112,17 +112,20 @@ void config_ddr(unsigned int pll, const struct ctrl_ioregs *ioregs,
 
 	/* Set CKE to be controlled by EMIF/DDR PHY */
 	writel(DDR_CKE_CTRL_NORMAL, &ddrctrl->ddrckectrl);
+
 #endif
 #ifdef CONFIG_AM43XX
 	writel(readl(&cm_device->cm_dll_ctrl) & ~0x1, &cm_device->cm_dll_ctrl);
 	while ((readl(&cm_device->cm_dll_ctrl) & CM_DLL_READYST) == 0)
 		;
-	writel(0x80000000, &ddrctrl->ddrioctrl);
 
 	config_io_ctrl(ioregs);
 
 	/* Set CKE to be controlled by EMIF/DDR PHY */
 	writel(DDR_CKE_CTRL_NORMAL, &ddrctrl->ddrckectrl);
+
+	/* Allow EMIF to control DDR_RESET */
+	writel(0x00000000, &ddrctrl->ddrioctrl);
 #endif
 
 	/* Program EMIF instance */
diff --git a/arch/arm/include/asm/arch-am33xx/cpu.h b/arch/arm/include/asm/arch-am33xx/cpu.h
index 8dd69b3c80..b94b56cba7 100644
--- a/arch/arm/include/asm/arch-am33xx/cpu.h
+++ b/arch/arm/include/asm/arch-am33xx/cpu.h
@@ -219,6 +219,12 @@ struct cm_dpll {
 	unsigned int resv4[2];
 	unsigned int clklcdcpixelclk;	/* offset 0x34 */
 };
+
+struct prm_device_inst {
+	unsigned int prm_rstctrl;
+	unsigned int prm_rsttime;
+	unsigned int prm_rstst;
+};
 #else
 /* Encapsulating core pll registers */
 struct cm_wkuppll {
@@ -386,6 +392,11 @@ struct cm_device_inst {
 	unsigned int cm_dll_ctrl;
 };
 
+struct prm_device_inst {
+	unsigned int prm_rstctrl;
+	unsigned int prm_rstst;
+};
+
 struct cm_dpll {
 	unsigned int resv1;
 	unsigned int clktimer2clk;	/* offset 0x04 */
diff --git a/arch/arm/include/asm/arch-am33xx/hardware_am33xx.h b/arch/arm/include/asm/arch-am33xx/hardware_am33xx.h
index c67a0801a9..d1aed58503 100644
--- a/arch/arm/include/asm/arch-am33xx/hardware_am33xx.h
+++ b/arch/arm/include/asm/arch-am33xx/hardware_am33xx.h
@@ -39,6 +39,7 @@
 /* VTP Base address */
 #define VTP0_CTRL_ADDR			0x44E10E0C
 #define VTP1_CTRL_ADDR			0x48140E10
+#define PRM_DEVICE_INST			0x44E00F00
 
 /* DDR Base address */
 #define DDR_PHY_CMD_ADDR		0x44E12000
diff --git a/arch/arm/include/asm/arch-am33xx/hardware_am43xx.h b/arch/arm/include/asm/arch-am33xx/hardware_am43xx.h
index efdecf4613..29e3816c1a 100644
--- a/arch/arm/include/asm/arch-am33xx/hardware_am43xx.h
+++ b/arch/arm/include/asm/arch-am33xx/hardware_am43xx.h
@@ -71,6 +71,7 @@
 #define PRM_PER_USBPHYOCP2SCP1_CLKCTRL (CM_PER + 0x5c0)
 #define USBPHYOCPSCP_MODULE_EN	(1 << 1)
 #define CM_DEVICE_INST			0x44df4100
+#define PRM_DEVICE_INST			0x44df4000
 
 /* Control status register */
 #define CTRL_CRYSTAL_FREQ_SRC_MASK		(1 << 31)
diff --git a/arch/arm/include/asm/emif.h b/arch/arm/include/asm/emif.h
index 2fe5776c6c..342f045f41 100644
--- a/arch/arm/include/asm/emif.h
+++ b/arch/arm/include/asm/emif.h
@@ -650,8 +650,8 @@ struct emif_reg_struct {
 	u32 emif_rd_wr_exec_thresh;
 	u32 emif_cos_config;
 	u32 padding9[6];
-	u32 emif_ddr_phy_status[21];
-	u32 padding10[27];
+	u32 emif_ddr_phy_status[28];
+	u32 padding10[20];
 	u32 emif_ddr_ext_phy_ctrl_1;
 	u32 emif_ddr_ext_phy_ctrl_1_shdw;
 	u32 emif_ddr_ext_phy_ctrl_2;
@@ -700,9 +700,36 @@ struct emif_reg_struct {
 	u32 emif_ddr_ext_phy_ctrl_23_shdw;
 	u32 emif_ddr_ext_phy_ctrl_24;
 	u32 emif_ddr_ext_phy_ctrl_24_shdw;
-	u32 padding[22];
-	u32 emif_ddr_fifo_misaligned_clear_1;
-	u32 emif_ddr_fifo_misaligned_clear_2;
+	u32 emif_ddr_ext_phy_ctrl_25;
+	u32 emif_ddr_ext_phy_ctrl_25_shdw;
+	u32 emif_ddr_ext_phy_ctrl_26;
+	u32 emif_ddr_ext_phy_ctrl_26_shdw;
+	u32 emif_ddr_ext_phy_ctrl_27;
+	u32 emif_ddr_ext_phy_ctrl_27_shdw;
+	u32 emif_ddr_ext_phy_ctrl_28;
+	u32 emif_ddr_ext_phy_ctrl_28_shdw;
+	u32 emif_ddr_ext_phy_ctrl_29;
+	u32 emif_ddr_ext_phy_ctrl_29_shdw;
+	u32 emif_ddr_ext_phy_ctrl_30;
+	u32 emif_ddr_ext_phy_ctrl_30_shdw;
+	u32 emif_ddr_ext_phy_ctrl_31;
+	u32 emif_ddr_ext_phy_ctrl_31_shdw;
+	u32 emif_ddr_ext_phy_ctrl_32;
+	u32 emif_ddr_ext_phy_ctrl_32_shdw;
+	u32 emif_ddr_ext_phy_ctrl_33;
+	u32 emif_ddr_ext_phy_ctrl_33_shdw;
+	u32 emif_ddr_ext_phy_ctrl_34;
+	u32 emif_ddr_ext_phy_ctrl_34_shdw;
+	u32 emif_ddr_ext_phy_ctrl_35;
+	u32 emif_ddr_ext_phy_ctrl_35_shdw;
+	union {
+		u32 emif_ddr_ext_phy_ctrl_36;
+		u32 emif_ddr_fifo_misaligned_clear_1;
+	};
+	union {
+		u32 emif_ddr_ext_phy_ctrl_36_shdw;
+		u32 emif_ddr_fifo_misaligned_clear_2;
+	};
 };
 
 struct dmm_lisa_map_regs {
diff --git a/board/ti/am43xx/board.c b/board/ti/am43xx/board.c
index 8695fc1790..d851f833aa 100644
--- a/board/ti/am43xx/board.c
+++ b/board/ti/am43xx/board.c
@@ -174,29 +174,6 @@ const struct emif_regs emif_regs_lpddr2 = {
 	.emif_cos_config			= 0x000FFFFF
 };
 
-const u32 ext_phy_ctrl_const_base_lpddr2[] = {
-	0x00500050,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x40001000,
-	0x08102040
-};
-
 const struct ctrl_ioregs ioregs_ddr3 = {
 	.cm0ioctl		= DDR3_ADDRCTRL_IOCTRL_VALUE,
 	.cm1ioctl		= DDR3_ADDRCTRL_WD0_IOCTRL_VALUE,
@@ -305,139 +282,6 @@ static const struct emif_regs ddr3_sk_emif_regs_400Mhz = {
 	.emif_cos_config		= 0x000FFFFF
 };
 
-const u32 ext_phy_ctrl_const_base_ddr3[] = {
-	0x00400040,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00340034,
-	0x00340034,
-	0x00340034,
-	0x00340034,
-	0x00340034,
-	0x0,
-	0x0,
-	0x40000000,
-	0x08102040
-};
-
-const u32 ext_phy_ctrl_const_base_ddr3_beta[] = {
-	0x00000000,
-	0x00000045,
-	0x00000046,
-	0x00000048,
-	0x00000047,
-	0x00000000,
-	0x0000004C,
-	0x00000070,
-	0x00000085,
-	0x000000A3,
-	0x00000000,
-	0x0000000C,
-	0x00000030,
-	0x00000045,
-	0x00000063,
-	0x00000000,
-	0x0,
-	0x0,
-	0x40000000,
-	0x08102040
-};
-
-const u32 ext_phy_ctrl_const_base_ddr3_production[] = {
-	0x00000000,
-	0x00000044,
-	0x00000044,
-	0x00000046,
-	0x00000046,
-	0x00000000,
-	0x00000059,
-	0x00000077,
-	0x00000093,
-	0x000000A8,
-	0x00000000,
-	0x00000019,
-	0x00000037,
-	0x00000053,
-	0x00000068,
-	0x00000000,
-	0x0,
-	0x0,
-	0x40000000,
-	0x08102040
-};
-
-static const u32 ext_phy_ctrl_const_base_ddr3_sk[] = {
-	/* first 5 are taken care by emif_regs */
-	0x00700070,
-
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-	0x00350035,
-
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-
-	0x00150015,
-	0x00150015,
-	0x00150015,
-	0x00150015,
-	0x00150015,
-
-	0x00800080,
-	0x00800080,
-
-	0x40000000,
-
-	0x08102040,
-
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-};
-
-void emif_get_ext_phy_ctrl_const_regs(const u32 **regs, u32 *size)
-{
-	if (board_is_eposevm()) {
-		*regs = ext_phy_ctrl_const_base_lpddr2;
-		*size = ARRAY_SIZE(ext_phy_ctrl_const_base_lpddr2);
-	} else if (board_is_evm_14_or_later()) {
-		*regs = ext_phy_ctrl_const_base_ddr3_production;
-		*size = ARRAY_SIZE(ext_phy_ctrl_const_base_ddr3_production);
-	} else if (board_is_evm_12_or_later()) {
-		*regs = ext_phy_ctrl_const_base_ddr3_beta;
-		*size = ARRAY_SIZE(ext_phy_ctrl_const_base_ddr3_beta);
-	} else if (board_is_gpevm()) {
-		*regs = ext_phy_ctrl_const_base_ddr3;
-		*size = ARRAY_SIZE(ext_phy_ctrl_const_base_ddr3);
-	} else if (board_is_sk()) {
-		*regs = ext_phy_ctrl_const_base_ddr3_sk;
-		*size = ARRAY_SIZE(ext_phy_ctrl_const_base_ddr3_sk);
-	}
-
-	return;
-}
-
 /*
  * get_sys_clk_index : returns the index of the sys_clk read from
  *			ctrl status register. This value is either