arm: mvebu: drivers/ddr: Add DDR3 driver with training code from Marvell bin_hdr

This patch adds the DDR3 setup and training code taken from the Marvell
U-Boot repository. This code used to be included as a binary (bin_hdr)
into the AXP boot image. Not linked with the main U-Boot. With this code
addition and the following serdes/PHY setup code, the Armada-XP support
in mainline U-Boot is finally self-contained. So the complete image
for booting can be built from mainline U-Boot. Without any additional
external inclusion. Hopefully other MVEBU SoC's will follow here.

Support for some SoC's has been removed in this version. This is:

MV_MSYS:
The code referred to by the MV_MSYS define is currently unused. And its
not really planned to support this in mainline. So lets remove it to
make the code clearer and increase the readability.

MV88F68XX (A38x):
The code referred to by the MV88F68XX define (A38x) is currently unused.
And its partial and not sufficient for this device in this stage.
So lets remove it to make the code clearer and increase the readability.

MV88F66XX (ALP):
The code referred to by the MV88F66XX define is currently unused. And its
not really planned to support this in mainline. So lets remove it to
make the code clearer and increase the readability.

MV88F78X60_Z1:
The code referred to by the MV88F78X60_Z1 define is currently unused. As the
Z1 revision of the AXP is not supported in mainline anymore.
So lets remove it to make the code clearer and increase the readability.

Remove support for Z1 & A0 AXP revisions (steppings). The current stepping
is B0 and this is the only one that is actively supported in this code
version.

Tested on AXP using a SPD DIMM setup on the Marvell DB-MV784MP-GP board and
on a custom fixed DDR configuration board (maxbcm).

Note:
This code has undergone many hours of coding-style cleanup and refactoring.
It still is not checkpatch clean though, I'm afraid. As the factoring of the
code has so many levels of indentation that many lines are longer than 80
chars. This might be some task to tackly later on.

Signed-off-by: Stefan Roese <sr@denx.de>
Reviewed-by: Luka Perkov <luka.perkov@sartura.hr>

1 files changed, 1214 insertions, 0 deletions

diff --git a/drivers/ddr/mvebu/ddr3_read_leveling.c b/drivers/ddr/mvebu/ddr3_read_leveling.c
new file mode 100644
index 0000000..4662bde
--- /dev/null
+++ b/drivers/ddr/mvebu/ddr3_read_leveling.c
@@ -0,0 +1,1214 @@
+/*
+ * Copyright (C) Marvell International Ltd. and its affiliates
+ *
+ * SPDX-License-Identifier:	GPL-2.0
+ */
+
+#include <common.h>
+#include <i2c.h>
+#include <spl.h>
+#include <asm/io.h>
+#include <asm/arch/cpu.h>
+#include <asm/arch/soc.h>
+
+#include "ddr3_hw_training.h"
+
+/*
+ * Debug
+ */
+#define DEBUG_RL_C(s, d, l) \
+	DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n")
+#define DEBUG_RL_FULL_C(s, d, l) \
+	DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n")
+
+#ifdef MV_DEBUG_RL
+#define DEBUG_RL_S(s) \
+	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s)
+#define DEBUG_RL_D(d, l) \
+	debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d)
+#else
+#define DEBUG_RL_S(s)
+#define DEBUG_RL_D(d, l)
+#endif
+
+#ifdef MV_DEBUG_RL_FULL
+#define DEBUG_RL_FULL_S(s)		puts(s)
+#define DEBUG_RL_FULL_D(d, l)		printf("%x", d)
+#else
+#define DEBUG_RL_FULL_S(s)
+#define DEBUG_RL_FULL_D(d, l)
+#endif
+
+extern u32 rl_pattern[LEN_STD_PATTERN];
+
+#ifdef RL_MODE
+static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
+						int ratio_2to1, u32 ecc,
+						MV_DRAM_INFO *dram_info);
+#else
+static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
+						    int ratio_2to1, u32 ecc,
+						    MV_DRAM_INFO *dram_info);
+#endif
+
+/*
+ * Name:     ddr3_read_leveling_hw
+ * Desc:     Execute the Read leveling phase by HW
+ * Args:     dram_info - main struct
+ *           freq      - current sequence frequency
+ * Notes:
+ * Returns:  MV_OK if success, MV_FAIL if fail.
+ */
+int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info)
+{
+	u32 reg;
+
+	/* Debug message - Start Read leveling procedure */
+	DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n");
+
+	/* Start Auto Read Leveling procedure */
+	reg = 1 << REG_DRAM_TRAINING_RL_OFFS;
+	/* Config the retest number */
+	reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS);
+
+	/* Enable CS in the automatic process */
+	reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS);
+
+	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
+
+	reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) |
+		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
+	reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg);
+
+	/* Wait */
+	do {
+		reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
+			(1 << REG_DRAM_TRAINING_AUTO_OFFS);
+	} while (reg);		/* Wait for '0' */
+
+	/* Check if Successful */
+	if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) &
+	    (1 << REG_DRAM_TRAINING_ERROR_OFFS)) {
+		u32 delay, phase, pup, cs;
+
+		dram_info->rl_max_phase = 0;
+		dram_info->rl_min_phase = 10;
+
+		/* Read results to arrays */
+		for (cs = 0; cs < MAX_CS; cs++) {
+			if (dram_info->cs_ena & (1 << cs)) {
+				for (pup = 0;
+				     pup < dram_info->num_of_total_pups;
+				     pup++) {
+					if (pup == dram_info->num_of_std_pups
+					    && dram_info->ecc_ena)
+						pup = ECC_PUP;
+					reg =
+					    ddr3_read_pup_reg(PUP_RL_MODE, cs,
+							      pup);
+					phase = (reg >> REG_PHY_PHASE_OFFS) &
+						PUP_PHASE_MASK;
+					delay = reg & PUP_DELAY_MASK;
+					dram_info->rl_val[cs][pup][P] = phase;
+					if (phase > dram_info->rl_max_phase)
+						dram_info->rl_max_phase = phase;
+					if (phase < dram_info->rl_min_phase)
+						dram_info->rl_min_phase = phase;
+					dram_info->rl_val[cs][pup][D] = delay;
+					dram_info->rl_val[cs][pup][S] =
+					    RL_FINAL_STATE;
+					reg =
+					    ddr3_read_pup_reg(PUP_RL_MODE + 0x1,
+							      cs, pup);
+					dram_info->rl_val[cs][pup][DQS] =
+					    (reg & 0x3F);
+				}
+#ifdef MV_DEBUG_RL
+				/* Print results */
+				DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ",
+					   (u32) cs, 1);
+
+				for (pup = 0;
+				     pup < (dram_info->num_of_total_pups);
+				     pup++) {
+					if (pup == dram_info->num_of_std_pups
+					    && dram_info->ecc_ena)
+						pup = ECC_PUP;
+					DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
+					DEBUG_RL_D((u32) pup, 1);
+					DEBUG_RL_S(", Phase: ");
+					DEBUG_RL_D((u32) dram_info->
+						   rl_val[cs][pup][P], 1);
+					DEBUG_RL_S(", Delay: ");
+					DEBUG_RL_D((u32) dram_info->
+						   rl_val[cs][pup][D], 2);
+					DEBUG_RL_S("\n");
+				}
+#endif
+			}
+		}
+
+		dram_info->rd_rdy_dly =
+			reg_read(REG_READ_DATA_READY_DELAYS_ADDR) &
+			REG_READ_DATA_SAMPLE_DELAYS_MASK;
+		dram_info->rd_smpl_dly =
+			reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) &
+			REG_READ_DATA_READY_DELAYS_MASK;
+#ifdef MV_DEBUG_RL
+		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
+			   dram_info->rd_smpl_dly, 2);
+		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
+			   dram_info->rd_rdy_dly, 2);
+		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n");
+#endif
+		return MV_OK;
+
+	} else {
+		DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n");
+		return MV_FAIL;
+	}
+}
+
+/*
+ * Name:     ddr3_read_leveling_sw
+ * Desc:     Execute the Read leveling phase by SW
+ * Args:     dram_info - main struct
+ *           freq      - current sequence frequency
+ * Notes:
+ * Returns:  MV_OK if success, MV_FAIL if fail.
+ */
+int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info)
+{
+	u32 reg, cs, ecc, pup_num, phase, delay, pup;
+	int status;
+
+	/* Debug message - Start Read leveling procedure */
+	DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n");
+
+	/* Enable SW Read Leveling */
+	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
+		(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
+	reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS);
+	/* [0]=1 - Enable SW override  */
+	/* 0x15B8 - Training SW 2 Register */
+	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+#ifdef RL_MODE
+	reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) |
+		(1 << REG_DRAM_TRAINING_AUTO_OFFS);
+	reg_write(REG_DRAM_TRAINING_ADDR, reg);	/* 0x15B0 - Training Register */
+#endif
+
+	/* Loop for each CS */
+	for (cs = 0; cs < dram_info->num_cs; cs++) {
+		DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1);
+
+		for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) {
+			/* ECC Support - Switch ECC Mux on ecc=1 */
+			reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
+				~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
+			reg |= (dram_info->ecc_ena *
+				ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
+			reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+			if (ecc)
+				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n");
+			else
+				DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n");
+
+			/* Set current sample delays */
+			reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
+			reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
+				 (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
+			reg |= (dram_info->cl <<
+				(REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs));
+			reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg);
+
+			/* Set current Ready delay */
+			reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
+			reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
+				 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+			if (!ratio_2to1) {
+				/* 1:1 mode */
+				reg |= ((dram_info->cl + 1) <<
+					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
+			} else {
+				/* 2:1 mode */
+				reg |= ((dram_info->cl + 2) <<
+					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
+			}
+			reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
+
+			/* Read leveling Single CS[cs] */
+#ifdef RL_MODE
+			status =
+			    ddr3_read_leveling_single_cs_rl_mode(cs, freq,
+								 ratio_2to1,
+								 ecc,
+								 dram_info);
+			if (MV_OK != status)
+				return status;
+#else
+			status =
+			    ddr3_read_leveling_single_cs_window_mode(cs, freq,
+								     ratio_2to1,
+								     ecc,
+								     dram_info)
+			    if (MV_OK != status)
+				return status;
+#endif
+		}
+
+		/* Print results */
+		DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs,
+			   1);
+
+		for (pup = 0;
+		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
+		     pup++) {
+			DEBUG_RL_S("DDR3 - Read Leveling - PUP: ");
+			DEBUG_RL_D((u32) pup, 1);
+			DEBUG_RL_S(", Phase: ");
+			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1);
+			DEBUG_RL_S(", Delay: ");
+			DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2);
+			DEBUG_RL_S("\n");
+		}
+
+		DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ",
+			   dram_info->rd_smpl_dly, 2);
+		DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ",
+			   dram_info->rd_rdy_dly, 2);
+
+		/* Configure PHY with average of 3 locked leveling settings */
+		for (pup = 0;
+		     pup < (dram_info->num_of_std_pups + dram_info->ecc_ena);
+		     pup++) {
+			/* ECC support - bit 8 */
+			pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup;
+
+			/* For now, set last cnt result */
+			phase = dram_info->rl_val[cs][pup][P];
+			delay = dram_info->rl_val[cs][pup][D];
+			ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase,
+					   delay);
+		}
+	}
+
+	/* Reset PHY read FIFO */
+	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
+		(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+	/* 0x15B8 - Training SW 2 Register */
+	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+	do {
+		reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
+			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+	} while (reg);		/* Wait for '0' */
+
+	/* ECC Support - Switch ECC Mux off ecc=0 */
+	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
+		~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS);
+	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+#ifdef RL_MODE
+	reg_write(REG_DRAM_TRAINING_ADDR, 0);	/* 0x15B0 - Training Register */
+#endif
+
+	/* Disable SW Read Leveling */
+	reg = reg_read(REG_DRAM_TRAINING_2_ADDR) &
+		~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS);
+	/* [0] = 0 - Disable SW override  */
+	reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS));
+	/* [3] = 1 - Disable RL MODE */
+	/* 0x15B8 - Training SW 2 Register */
+	reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+	DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n");
+	return MV_OK;
+}
+
+#ifdef RL_MODE
+/*
+ * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode().
+ * This just got too much indented making it hard to read / edit.
+ */
+static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups,
+		    u32 *locked_sum, u32 ecc, int *first_octet_locked,
+		    int *counter_in_progress, int final_delay, u32 delay,
+		    u32 phase)
+{
+	/* If no OverRun */
+	if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) {
+		int idx;
+
+		idx = pup + ecc * ECC_BIT;
+
+		/* PUP passed, start examining */
+		if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) {
+			/* Must be RL_UNLOCK_STATE */
+			/* Match expected value ? - Update State Machine */
+			if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) {
+				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
+						(u32)pup, 1);
+				info->rl_val[cs][idx][C]++;
+
+				/* If pup got to last state - lock the delays */
+				if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) {
+					info->rl_val[cs][idx][C] = 0;
+					info->rl_val[cs][idx][DS] = delay;
+					info->rl_val[cs][idx][PS] = phase;
+
+					/* Go to Final State */
+					info->rl_val[cs][idx][S] = RL_FINAL_STATE;
+					*locked_sum = *locked_sum + 1;
+					DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ",
+							(u32)pup, 1);
+
+					/*
+					 * If first lock - need to lock delays
+					 */
+					if (*first_octet_locked == 0) {
+						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
+								(u32)pup, 1);
+						*first_octet_locked = 1;
+					}
+
+					/*
+					 * If pup is in not in final state but
+					 * there was match - dont increment
+					 * counter
+					 */
+				} else {
+					*counter_in_progress = 1;
+				}
+			}
+		}
+	}
+}
+
+/*
+ * Name:     ddr3_read_leveling_single_cs_rl_mode
+ * Desc:     Execute Read leveling for single Chip select
+ * Args:     cs        - current chip select
+ *           freq      - current sequence frequency
+ *           ecc       - ecc iteration indication
+ *           dram_info - main struct
+ * Notes:
+ * Returns:  MV_OK if success, MV_FAIL if fail.
+ */
+static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq,
+						int ratio_2to1, u32 ecc,
+						MV_DRAM_INFO *dram_info)
+{
+	u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups,
+		repeat_max_cnt, sdram_offset, locked_sum;
+	u32 phase_min, ui_max_delay;
+	int all_locked, first_octet_locked, counter_in_progress;
+	int final_delay = 0;
+
+	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
+
+	/* Init values */
+	phase = 0;
+	delay = 0;
+	rd_sample_delay = dram_info->cl;
+	all_locked = 0;
+	first_octet_locked = 0;
+	repeat_max_cnt = 0;
+	locked_sum = 0;
+
+	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
+	     pup++)
+		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
+
+	/* Main loop */
+	while (!all_locked) {
+		counter_in_progress = 0;
+
+		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
+		DEBUG_RL_FULL_D(rd_sample_delay, 2);
+		DEBUG_RL_FULL_S(", RdRdyDly = ");
+		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
+		DEBUG_RL_FULL_S(", Phase = ");
+		DEBUG_RL_FULL_D(phase, 1);
+		DEBUG_RL_FULL_S(", Delay = ");
+		DEBUG_RL_FULL_D(delay, 2);
+		DEBUG_RL_FULL_S("\n");
+
+		/*
+		 * Broadcast to all PUPs current RL delays: DQS phase,
+		 * leveling delay
+		 */
+		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
+
+		/* Reset PHY read FIFO */
+		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
+			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+		/* 0x15B8 - Training SW 2 Register */
+		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+		do {
+			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
+				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+		} while (reg);	/* Wait for '0' */
+
+		/* Read pattern from SDRAM */
+		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
+		locked_pups = 0;
+		if (MV_OK !=
+		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
+				       rl_pattern, LEN_STD_PATTERN,
+				       sdram_offset, 0, 0, NULL, 0))
+			return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN;
+
+		/* Octet evaluation */
+		/* pup_num = Q or 1 for ECC */
+		for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
+			/* Check Overrun */
+			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
+			       (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) {
+				overrun(cs, dram_info, pup, locked_pups,
+					&locked_sum, ecc, &first_octet_locked,
+					&counter_in_progress, final_delay,
+					delay, phase);
+			} else {
+				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
+						(u32)pup, 1);
+			}
+		}
+
+		if (locked_sum == (dram_info->num_of_std_pups *
+				   (1 - ecc) + ecc)) {
+			all_locked = 1;
+			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
+		}
+
+		/*
+		 * This is a fix for unstable condition where pups are
+		 * toggling between match and no match
+		 */
+		/*
+		 * If some of the pups is >1 <3, check if we did it too
+		 * many times
+		 */
+		if (counter_in_progress == 1) {
+			/* Notify at least one Counter is >=1 and < 3 */
+			if (repeat_max_cnt < RL_RETRY_COUNT) {
+				repeat_max_cnt++;
+				counter_in_progress = 1;
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
+			} else {
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
+				counter_in_progress = 0;
+			}
+		}
+
+		/*
+		 * Check some of the pups are in the middle of state machine
+		 * and don't increment the delays
+		 */
+		if (!counter_in_progress && !all_locked) {
+			int idx;
+
+			idx = pup + ecc * ECC_BIT;
+
+			repeat_max_cnt = 0;
+			/* if 1:1 mode */
+			if ((!ratio_2to1) && ((phase == 0) || (phase == 4)))
+				ui_max_delay = MAX_DELAY_INV;
+			else
+				ui_max_delay = MAX_DELAY;
+
+			/* Increment Delay */
+			if (delay < ui_max_delay) {
+				delay++;
+				/*
+				 * Mark the last delay/pahse place for
+				 * window final place
+				 */
+				if (delay == ui_max_delay) {
+					if ((!ratio_2to1 && phase ==
+					     MAX_PHASE_RL_L_1TO1)
+					    || (ratio_2to1 && phase ==
+						MAX_PHASE_RL_L_2TO1))
+						final_delay = 1;
+				}
+			} else {
+				/* Phase+CL Incrementation */
+				delay = 0;
+
+				if (!ratio_2to1) {
+					/* 1:1 mode */
+					if (first_octet_locked) {
+						/* some Pup was Locked */
+						if (phase < MAX_PHASE_RL_L_1TO1) {
+							if (phase == 1) {
+								phase = 4;
+							} else {
+								phase++;
+								delay = MIN_DELAY_PHASE_1_LIMIT;
+							}
+						} else {
+							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+							DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n");
+							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
+						}
+					} else {
+						/* NO Pup was Locked */
+						if (phase < MAX_PHASE_RL_UL_1TO1) {
+							phase++;
+							delay =
+							    MIN_DELAY_PHASE_1_LIMIT;
+						} else {
+							phase = 0;
+						}
+					}
+				} else {
+					/* 2:1 mode */
+					if (first_octet_locked) {
+						/* some Pup was Locked */
+						if (phase < MAX_PHASE_RL_L_2TO1) {
+							phase++;
+						} else {
+							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+							DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+							for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
+								/* pup_num = Q or 1 for ECC */
+								if (dram_info->rl_val[cs][idx][S]
+								    == 0) {
+									DEBUG_RL_C("Failed byte is = ",
+										   pup, 1);
+								}
+							}
+							return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK;
+						}
+					} else {
+						/* No Pup was Locked */
+						if (phase < MAX_PHASE_RL_UL_2TO1)
+							phase++;
+						else
+							phase = 0;
+					}
+				}
+
+				/*
+				 * If we finished a full Phases cycle (so now
+				 * phase = 0, need to increment rd_sample_dly
+				 */
+				if (phase == 0 && first_octet_locked == 0) {
+					rd_sample_delay++;
+					if (rd_sample_delay == 0x10) {
+						DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+						DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+						for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) {
+							/* pup_num = Q or 1 for ECC */
+							if (dram_info->
+							    rl_val[cs][idx][S] == 0) {
+								DEBUG_RL_C("Failed byte is = ",
+									   pup, 1);
+							}
+						}
+						return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK;
+					}
+
+					/* Set current rd_sample_delay  */
+					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
+					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK
+					      << (REG_READ_DATA_SAMPLE_DELAYS_OFFS
+						  * cs));
+					reg |= (rd_sample_delay <<
+						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
+						 cs));
+					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
+						  reg);
+				}
+
+				/*
+				 * Set current rdReadyDelay according to the
+				 * hash table (Need to do this in every phase
+				 * change)
+				 */
+				if (!ratio_2to1) {
+					/* 1:1 mode */
+					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
+					switch (phase) {
+					case 0:
+						add = (add >>
+						       REG_TRAINING_DEBUG_2_OFFS);
+						break;
+					case 1:
+						add = (add >>
+						       (REG_TRAINING_DEBUG_2_OFFS
+							+ 3));
+						break;
+					case 4:
+						add = (add >>
+						       (REG_TRAINING_DEBUG_2_OFFS
+							+ 6));
+						break;
+					case 5:
+						add = (add >>
+						       (REG_TRAINING_DEBUG_2_OFFS
+							+ 9));
+						break;
+					}
+					add &= REG_TRAINING_DEBUG_2_MASK;
+				} else {
+					/* 2:1 mode */
+					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
+					add = (add >>
+					       (phase *
+						REG_TRAINING_DEBUG_3_OFFS));
+					add &= REG_TRAINING_DEBUG_3_MASK;
+				}
+
+				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
+				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
+					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+				reg |= ((rd_sample_delay + add) <<
+					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
+				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
+				dram_info->rd_smpl_dly = rd_sample_delay;
+				dram_info->rd_rdy_dly = rd_sample_delay + add;
+			}
+
+			/* Reset counters for pups with states<RD_STATE_COUNT */
+			for (pup = 0; pup <
+				     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
+			     pup++) {
+				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
+					dram_info->rl_val[cs][idx][C] = 0;
+			}
+		}
+	}
+
+	phase_min = 10;
+
+	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
+		if (dram_info->rl_val[cs][pup][PS] < phase_min)
+			phase_min = dram_info->rl_val[cs][pup][PS];
+	}
+
+	/*
+	 * Set current rdReadyDelay according to the hash table (Need to
+	 * do this in every phase change)
+	 */
+	if (!ratio_2to1) {
+		/* 1:1 mode */
+		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
+		switch (phase_min) {
+		case 0:
+			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
+			break;
+		case 1:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
+			break;
+		case 4:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
+			break;
+		case 5:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
+			break;
+		}
+		add &= REG_TRAINING_DEBUG_2_MASK;
+	} else {
+		/* 2:1 mode */
+		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
+		add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS));
+		add &= REG_TRAINING_DEBUG_3_MASK;
+	}
+
+	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
+	reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
+		 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+	reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
+	dram_info->rd_rdy_dly = rd_sample_delay + add;
+
+	for (cs = 0; cs < dram_info->num_cs; cs++) {
+		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
+			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
+			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
+		}
+	}
+
+	return MV_OK;
+}
+
+#else
+
+/*
+ * Name:     ddr3_read_leveling_single_cs_window_mode
+ * Desc:     Execute Read leveling for single Chip select
+ * Args:     cs        - current chip select
+ *           freq      - current sequence frequency
+ *           ecc       - ecc iteration indication
+ *           dram_info - main struct
+ * Notes:
+ * Returns:  MV_OK if success, MV_FAIL if fail.
+ */
+static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq,
+						    int ratio_2to1, u32 ecc,
+						    MV_DRAM_INFO *dram_info)
+{
+	u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups,
+	    repeat_max_cnt, sdram_offset, final_sum, locked_sum;
+	u32 delay_s, delay_e, tmp, phase_min, ui_max_delay;
+	int all_locked, first_octet_locked, counter_in_progress;
+	int final_delay = 0;
+
+	DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1);
+
+	/* Init values */
+	phase = 0;
+	delay = 0;
+	rd_sample_delay = dram_info->cl;
+	all_locked = 0;
+	first_octet_locked = 0;
+	repeat_max_cnt = 0;
+	sum = 0;
+	final_sum = 0;
+	locked_sum = 0;
+
+	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
+	     pup++)
+		dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0;
+
+	/* Main loop */
+	while (!all_locked) {
+		counter_in_progress = 0;
+
+		DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = ");
+		DEBUG_RL_FULL_D(rd_sample_delay, 2);
+		DEBUG_RL_FULL_S(", RdRdyDly = ");
+		DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2);
+		DEBUG_RL_FULL_S(", Phase = ");
+		DEBUG_RL_FULL_D(phase, 1);
+		DEBUG_RL_FULL_S(", Delay = ");
+		DEBUG_RL_FULL_D(delay, 2);
+		DEBUG_RL_FULL_S("\n");
+
+		/*
+		 * Broadcast to all PUPs current RL delays: DQS phase,leveling
+		 * delay
+		 */
+		ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay);
+
+		/* Reset PHY read FIFO */
+		reg = reg_read(REG_DRAM_TRAINING_2_ADDR) |
+			(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+		/* 0x15B8 - Training SW 2 Register */
+		reg_write(REG_DRAM_TRAINING_2_ADDR, reg);
+
+		do {
+			reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) &
+				(1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS);
+		} while (reg);	/* Wait for '0' */
+
+		/* Read pattern from SDRAM */
+		sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS;
+		locked_pups = 0;
+		if (MV_OK !=
+		    ddr3_sdram_compare(dram_info, 0xFF, &locked_pups,
+				       rl_pattern, LEN_STD_PATTERN,
+				       sdram_offset, 0, 0, NULL, 0))
+			return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN;
+
+		/* Octet evaluation */
+		for (pup = 0; pup < (dram_info->num_of_std_pups *
+				     (1 - ecc) + ecc); pup++) {
+			/* pup_num = Q or 1 for ECC */
+			int idx;
+
+			idx = pup + ecc * ECC_BIT;
+
+			/* Check Overrun */
+			if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >>
+			      (REG_DRAM_TRAINING_2_OVERRUN_OFFS +
+			       pup)) & 0x1)) {
+				/* If no OverRun */
+
+				/* Inside the window */
+				if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) {
+					/*
+					 * Match expected value ? - Update
+					 * State Machine
+					 */
+					if (((~locked_pups >> pup) & 0x1)
+					    && (final_delay == 0)) {
+						/* Match - Still inside the Window */
+						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window  for pup: ",
+								(u32)pup, 1);
+
+					} else {
+						/* We got fail -> this is the end of the window */
+						dram_info->rl_val[cs][idx][DE] = delay;
+						dram_info->rl_val[cs][idx][PE] = phase;
+						/* Go to Final State */
+						dram_info->rl_val[cs][idx][S]++;
+						final_sum++;
+						DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ",
+								(u32)pup, 1);
+					}
+
+					/* Before the start of the window */
+				} else if (dram_info->rl_val[cs][idx][S] ==
+					   RL_UNLOCK_STATE) {
+					/* Must be RL_UNLOCK_STATE */
+					/*
+					 * Match expected value ? - Update
+					 * State Machine
+					 */
+					if (dram_info->rl_val[cs][idx][C] <
+					    RL_RETRY_COUNT) {
+						if (((~locked_pups >> pup) & 0x1)) {
+							/* Match */
+							DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ",
+									(u32)pup, 1);
+							dram_info->rl_val[cs][idx][C]++;
+
+							/* If pup got to last state - lock the delays */
+							if (dram_info->rl_val[cs][idx][C] ==
+							    RL_RETRY_COUNT) {
+								dram_info->rl_val[cs][idx][C] = 0;
+								dram_info->rl_val[cs][idx][DS] =
+									delay;
+								dram_info->rl_val[cs][idx][PS] =
+									phase;
+								dram_info->rl_val[cs][idx][S]++;	/* Go to Window State */
+								locked_sum++;
+								/* Will count the pups that got locked */
+
+								/* IF First lock - need to lock delays */
+								if (first_octet_locked == 0) {
+									DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ",
+											(u32)pup, 1);
+									first_octet_locked
+									    =
+									    1;
+								}
+							}
+
+							/* if pup is in not in final state but there was match - dont increment counter */
+							else {
+								counter_in_progress
+								    = 1;
+							}
+						}
+					}
+				}
+			} else {
+				DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ",
+						(u32)pup, 1);
+				counter_in_progress = 1;
+			}
+		}
+
+		if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) {
+			all_locked = 1;
+			DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n");
+		}
+
+		/*
+		 * This is a fix for unstable condition where pups are
+		 * toggling between match and no match
+		 */
+		/*
+		 * If some of the pups is >1 <3, check if we did it too many
+		 * times
+		 */
+		if (counter_in_progress == 1) {
+			if (repeat_max_cnt < RL_RETRY_COUNT) {
+				/* Notify at least one Counter is >=1 and < 3 */
+				repeat_max_cnt++;
+				counter_in_progress = 1;
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n");
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n");
+			} else {
+				DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n");
+				counter_in_progress = 0;
+			}
+		}
+
+		/*
+		 * Check some of the pups are in the middle of state machine
+		 * and don't increment the delays
+		 */
+		if (!counter_in_progress && !all_locked) {
+			repeat_max_cnt = 0;
+			if (!ratio_2to1)
+				ui_max_delay = MAX_DELAY_INV;
+			else
+				ui_max_delay = MAX_DELAY;
+
+			/* Increment Delay */
+			if (delay < ui_max_delay) {
+				/* Delay Incrementation */
+				delay++;
+				if (delay == ui_max_delay) {
+					/*
+					 * Mark the last delay/pahse place
+					 * for window final place
+					 */
+					if ((!ratio_2to1
+					     && phase == MAX_PHASE_RL_L_1TO1)
+					    || (ratio_2to1
+						&& phase ==
+						MAX_PHASE_RL_L_2TO1))
+						final_delay = 1;
+				}
+			} else {
+				/* Phase+CL Incrementation */
+				delay = 0;
+				if (!ratio_2to1) {
+					/* 1:1 mode */
+					if (first_octet_locked) {
+						/* some pupet was Locked */
+						if (phase < MAX_PHASE_RL_L_1TO1) {
+#ifdef RL_WINDOW_WA
+							if (phase == 0)
+#else
+							if (phase == 1)
+#endif
+								phase = 4;
+							else
+								phase++;
+						} else {
+							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
+						}
+					} else {
+						/* No Pup was Locked */
+						if (phase < MAX_PHASE_RL_UL_1TO1) {
+#ifdef RL_WINDOW_WA
+							if (phase == 0)
+								phase = 4;
+#else
+							phase++;
+#endif
+						} else
+							phase = 0;
+					}
+				} else {
+					/* 2:1 mode */
+					if (first_octet_locked) {
+						/* Some Pup was Locked */
+						if (phase < MAX_PHASE_RL_L_2TO1) {
+							phase++;
+						} else {
+							DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n");
+							return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK;
+						}
+					} else {
+						/* No Pup was Locked */
+						if (phase < MAX_PHASE_RL_UL_2TO1)
+							phase++;
+						else
+							phase = 0;
+					}
+				}
+
+				/*
+				 * If we finished a full Phases cycle (so
+				 * now phase = 0, need to increment
+				 * rd_sample_dly
+				 */
+				if (phase == 0 && first_octet_locked == 0) {
+					rd_sample_delay++;
+
+					/* Set current rd_sample_delay  */
+					reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR);
+					reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK <<
+						 (REG_READ_DATA_SAMPLE_DELAYS_OFFS
+						  * cs));
+					reg |= (rd_sample_delay <<
+						(REG_READ_DATA_SAMPLE_DELAYS_OFFS *
+						 cs));
+					reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR,
+						  reg);
+				}
+
+				/*
+				 * Set current rdReadyDelay according to the
+				 * hash table (Need to do this in every phase
+				 * change)
+				 */
+				if (!ratio_2to1) {
+					/* 1:1 mode */
+					add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
+					switch (phase) {
+					case 0:
+						add = add >>
+							REG_TRAINING_DEBUG_2_OFFS;
+						break;
+					case 1:
+						add = add >>
+							(REG_TRAINING_DEBUG_2_OFFS
+							 + 3);
+						break;
+					case 4:
+						add = add >>
+							(REG_TRAINING_DEBUG_2_OFFS
+							 + 6);
+						break;
+					case 5:
+						add = add >>
+							(REG_TRAINING_DEBUG_2_OFFS
+							 + 9);
+						break;
+					}
+				} else {
+					/* 2:1 mode */
+					add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
+					add = (add >> phase *
+					       REG_TRAINING_DEBUG_3_OFFS);
+				}
+				add &= REG_TRAINING_DEBUG_2_MASK;
+				reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
+				reg &= ~(REG_READ_DATA_READY_DELAYS_MASK <<
+					 (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+				reg |= ((rd_sample_delay + add) <<
+					(REG_READ_DATA_READY_DELAYS_OFFS * cs));
+				reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
+				dram_info->rd_smpl_dly = rd_sample_delay;
+				dram_info->rd_rdy_dly = rd_sample_delay + add;
+			}
+
+			/* Reset counters for pups with states<RD_STATE_COUNT */
+			for (pup = 0;
+			     pup <
+			     (dram_info->num_of_std_pups * (1 - ecc) + ecc);
+			     pup++) {
+				if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT)
+					dram_info->rl_val[cs][idx][C] = 0;
+			}
+		}
+	}
+
+	phase_min = 10;
+
+	for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) {
+		DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: ");
+		DEBUG_RL_D((u32) pup, 1);
+		DEBUG_RL_S(", PS: ");
+		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1);
+		DEBUG_RL_S(", DS: ");
+		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2);
+		DEBUG_RL_S(", PE: ");
+		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1);
+		DEBUG_RL_S(", DE: ");
+		DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2);
+		DEBUG_RL_S("\n");
+	}
+
+	/* Find center of the window procedure */
+	for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc);
+	     pup++) {
+#ifdef RL_WINDOW_WA
+		if (!ratio_2to1) {	/* 1:1 mode */
+			if (dram_info->rl_val[cs][idx][PS] == 4)
+				dram_info->rl_val[cs][idx][PS] = 1;
+			if (dram_info->rl_val[cs][idx][PE] == 4)
+				dram_info->rl_val[cs][idx][PE] = 1;
+
+			delay_s = dram_info->rl_val[cs][idx][PS] *
+				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS];
+			delay_e = dram_info->rl_val[cs][idx][PE] *
+				MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE];
+
+			tmp = (delay_e - delay_s) / 2 + delay_s;
+			phase = tmp / MAX_DELAY_INV;
+			if (phase == 1)	/* 1:1 mode */
+				phase = 4;
+
+			if (phase < phase_min)	/* for the read ready delay */
+				phase_min = phase;
+
+			dram_info->rl_val[cs][idx][P] = phase;
+			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV;
+
+		} else {
+			delay_s = dram_info->rl_val[cs][idx][PS] *
+				MAX_DELAY + dram_info->rl_val[cs][idx][DS];
+			delay_e = dram_info->rl_val[cs][idx][PE] *
+				MAX_DELAY + dram_info->rl_val[cs][idx][DE];
+
+			tmp = (delay_e - delay_s) / 2 + delay_s;
+			phase = tmp / MAX_DELAY;
+
+			if (phase < phase_min)	/* for the read ready delay */
+				phase_min = phase;
+
+			dram_info->rl_val[cs][idx][P] = phase;
+			dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
+		}
+#else
+		if (!ratio_2to1) {	/* 1:1 mode */
+			if (dram_info->rl_val[cs][idx][PS] > 1)
+				dram_info->rl_val[cs][idx][PS] -= 2;
+			if (dram_info->rl_val[cs][idx][PE] > 1)
+				dram_info->rl_val[cs][idx][PE] -= 2;
+		}
+
+		delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY +
+			dram_info->rl_val[cs][idx][DS];
+		delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY +
+			dram_info->rl_val[cs][idx][DE];
+
+		tmp = (delay_e - delay_s) / 2 + delay_s;
+		phase = tmp / MAX_DELAY;
+		if (!ratio_2to1 && phase > 1)	/* 1:1 mode */
+			phase += 2;
+
+		if (phase < phase_min)	/* for the read ready delay */
+			phase_min = phase;
+
+		dram_info->rl_val[cs][idx][P] = phase;
+		dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY;
+#endif
+	}
+
+	/* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */
+	if (!ratio_2to1) {	/* 1:1 mode */
+		add = reg_read(REG_TRAINING_DEBUG_2_ADDR);
+		switch (phase_min) {
+		case 0:
+			add = (add >> REG_TRAINING_DEBUG_2_OFFS);
+			break;
+		case 1:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3));
+			break;
+		case 4:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6));
+			break;
+		case 5:
+			add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9));
+			break;
+		}
+	} else {		/* 2:1 mode */
+		add = reg_read(REG_TRAINING_DEBUG_3_ADDR);
+		add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS);
+	}
+
+	add &= REG_TRAINING_DEBUG_2_MASK;
+	reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR);
+	reg &=
+	    ~(REG_READ_DATA_READY_DELAYS_MASK <<
+	      (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+	reg |=
+	    ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs));
+	reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg);
+	dram_info->rd_rdy_dly = rd_sample_delay + add;
+
+	for (cs = 0; cs < dram_info->num_cs; cs++) {
+		for (pup = 0; pup < dram_info->num_of_total_pups; pup++) {
+			reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup);
+			dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F);
+		}
+	}
+
+	return MV_OK;
+}
+#endif