/* * Copyright (C) Marvell International Ltd. and its affiliates * * SPDX-License-Identifier: GPL-2.0 */ #include #include #include #include #include #include #include "ddr3_hw_training.h" /* * Debug */ #define DEBUG_RL_C(s, d, l) \ DEBUG_RL_S(s); DEBUG_RL_D(d, l); DEBUG_RL_S("\n") #define DEBUG_RL_FULL_C(s, d, l) \ DEBUG_RL_FULL_S(s); DEBUG_RL_FULL_D(d, l); DEBUG_RL_FULL_S("\n") #ifdef MV_DEBUG_RL #define DEBUG_RL_S(s) \ debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%s", s) #define DEBUG_RL_D(d, l) \ debug_cond(ddr3_get_log_level() >= MV_LOG_LEVEL_2, "%x", d) #else #define DEBUG_RL_S(s) #define DEBUG_RL_D(d, l) #endif #ifdef MV_DEBUG_RL_FULL #define DEBUG_RL_FULL_S(s) puts(s) #define DEBUG_RL_FULL_D(d, l) printf("%x", d) #else #define DEBUG_RL_FULL_S(s) #define DEBUG_RL_FULL_D(d, l) #endif extern u32 rl_pattern[LEN_STD_PATTERN]; #ifdef RL_MODE static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq, int ratio_2to1, u32 ecc, MV_DRAM_INFO *dram_info); #else static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq, int ratio_2to1, u32 ecc, MV_DRAM_INFO *dram_info); #endif /* * Name: ddr3_read_leveling_hw * Desc: Execute the Read leveling phase by HW * Args: dram_info - main struct * freq - current sequence frequency * Notes: * Returns: MV_OK if success, MV_FAIL if fail. */ int ddr3_read_leveling_hw(u32 freq, MV_DRAM_INFO *dram_info) { u32 reg; /* Debug message - Start Read leveling procedure */ DEBUG_RL_S("DDR3 - Read Leveling - Starting HW RL procedure\n"); /* Start Auto Read Leveling procedure */ reg = 1 << REG_DRAM_TRAINING_RL_OFFS; /* Config the retest number */ reg |= (COUNT_HW_RL << REG_DRAM_TRAINING_RETEST_OFFS); /* Enable CS in the automatic process */ reg |= (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS); reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) | (1 << REG_DRAM_TRAINING_AUTO_OFFS); reg_write(REG_DRAM_TRAINING_SHADOW_ADDR, reg); /* Wait */ do { reg = reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) & (1 << REG_DRAM_TRAINING_AUTO_OFFS); } while (reg); /* Wait for '0' */ /* Check if Successful */ if (reg_read(REG_DRAM_TRAINING_SHADOW_ADDR) & (1 << REG_DRAM_TRAINING_ERROR_OFFS)) { u32 delay, phase, pup, cs; dram_info->rl_max_phase = 0; dram_info->rl_min_phase = 10; /* Read results to arrays */ for (cs = 0; cs < MAX_CS; cs++) { if (dram_info->cs_ena & (1 << cs)) { for (pup = 0; pup < dram_info->num_of_total_pups; pup++) { if (pup == dram_info->num_of_std_pups && dram_info->ecc_ena) pup = ECC_PUP; reg = ddr3_read_pup_reg(PUP_RL_MODE, cs, pup); phase = (reg >> REG_PHY_PHASE_OFFS) & PUP_PHASE_MASK; delay = reg & PUP_DELAY_MASK; dram_info->rl_val[cs][pup][P] = phase; if (phase > dram_info->rl_max_phase) dram_info->rl_max_phase = phase; if (phase < dram_info->rl_min_phase) dram_info->rl_min_phase = phase; dram_info->rl_val[cs][pup][D] = delay; dram_info->rl_val[cs][pup][S] = RL_FINAL_STATE; reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup); dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F); } #ifdef MV_DEBUG_RL /* Print results */ DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs, 1); for (pup = 0; pup < (dram_info->num_of_total_pups); pup++) { if (pup == dram_info->num_of_std_pups && dram_info->ecc_ena) pup = ECC_PUP; DEBUG_RL_S("DDR3 - Read Leveling - PUP: "); DEBUG_RL_D((u32) pup, 1); DEBUG_RL_S(", Phase: "); DEBUG_RL_D((u32) dram_info-> rl_val[cs][pup][P], 1); DEBUG_RL_S(", Delay: "); DEBUG_RL_D((u32) dram_info-> rl_val[cs][pup][D], 2); DEBUG_RL_S("\n"); } #endif } } dram_info->rd_rdy_dly = reg_read(REG_READ_DATA_READY_DELAYS_ADDR) & REG_READ_DATA_SAMPLE_DELAYS_MASK; dram_info->rd_smpl_dly = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR) & REG_READ_DATA_READY_DELAYS_MASK; #ifdef MV_DEBUG_RL DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ", dram_info->rd_smpl_dly, 2); DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ", dram_info->rd_rdy_dly, 2); DEBUG_RL_S("DDR3 - Read Leveling - HW RL Ended Successfully\n"); #endif return MV_OK; } else { DEBUG_RL_S("DDR3 - Read Leveling - HW RL Error\n"); return MV_FAIL; } } /* * Name: ddr3_read_leveling_sw * Desc: Execute the Read leveling phase by SW * Args: dram_info - main struct * freq - current sequence frequency * Notes: * Returns: MV_OK if success, MV_FAIL if fail. */ int ddr3_read_leveling_sw(u32 freq, int ratio_2to1, MV_DRAM_INFO *dram_info) { u32 reg, cs, ecc, pup_num, phase, delay, pup; int status; /* Debug message - Start Read leveling procedure */ DEBUG_RL_S("DDR3 - Read Leveling - Starting SW RL procedure\n"); /* Enable SW Read Leveling */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | (1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); reg &= ~(1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS); /* [0]=1 - Enable SW override */ /* 0x15B8 - Training SW 2 Register */ reg_write(REG_DRAM_TRAINING_2_ADDR, reg); #ifdef RL_MODE reg = (dram_info->cs_ena << REG_DRAM_TRAINING_CS_OFFS) | (1 << REG_DRAM_TRAINING_AUTO_OFFS); reg_write(REG_DRAM_TRAINING_ADDR, reg); /* 0x15B0 - Training Register */ #endif /* Loop for each CS */ for (cs = 0; cs < dram_info->num_cs; cs++) { DEBUG_RL_C("DDR3 - Read Leveling - CS - ", (u32) cs, 1); for (ecc = 0; ecc <= (dram_info->ecc_ena); ecc++) { /* ECC Support - Switch ECC Mux on ecc=1 */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); reg |= (dram_info->ecc_ena * ecc << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); reg_write(REG_DRAM_TRAINING_2_ADDR, reg); if (ecc) DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Enabled\n"); else DEBUG_RL_S("DDR3 - Read Leveling - ECC Mux Disabled\n"); /* Set current sample delays */ reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg |= (dram_info->cl << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg); /* Set current Ready delay */ reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); if (!ratio_2to1) { /* 1:1 mode */ reg |= ((dram_info->cl + 1) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); } else { /* 2:1 mode */ reg |= ((dram_info->cl + 2) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); } reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); /* Read leveling Single CS[cs] */ #ifdef RL_MODE status = ddr3_read_leveling_single_cs_rl_mode(cs, freq, ratio_2to1, ecc, dram_info); if (MV_OK != status) return status; #else status = ddr3_read_leveling_single_cs_window_mode(cs, freq, ratio_2to1, ecc, dram_info) if (MV_OK != status) return status; #endif } /* Print results */ DEBUG_RL_C("DDR3 - Read Leveling - Results for CS - ", (u32) cs, 1); for (pup = 0; pup < (dram_info->num_of_std_pups + dram_info->ecc_ena); pup++) { DEBUG_RL_S("DDR3 - Read Leveling - PUP: "); DEBUG_RL_D((u32) pup, 1); DEBUG_RL_S(", Phase: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][P], 1); DEBUG_RL_S(", Delay: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][D], 2); DEBUG_RL_S("\n"); } DEBUG_RL_C("DDR3 - Read Leveling - Read Sample Delay: ", dram_info->rd_smpl_dly, 2); DEBUG_RL_C("DDR3 - Read Leveling - Read Ready Delay: ", dram_info->rd_rdy_dly, 2); /* Configure PHY with average of 3 locked leveling settings */ for (pup = 0; pup < (dram_info->num_of_std_pups + dram_info->ecc_ena); pup++) { /* ECC support - bit 8 */ pup_num = (pup == dram_info->num_of_std_pups) ? ECC_BIT : pup; /* For now, set last cnt result */ phase = dram_info->rl_val[cs][pup][P]; delay = dram_info->rl_val[cs][pup][D]; ddr3_write_pup_reg(PUP_RL_MODE, cs, pup_num, phase, delay); } } /* Reset PHY read FIFO */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); /* 0x15B8 - Training SW 2 Register */ reg_write(REG_DRAM_TRAINING_2_ADDR, reg); do { reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); } while (reg); /* Wait for '0' */ /* ECC Support - Switch ECC Mux off ecc=0 */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & ~(1 << REG_DRAM_TRAINING_2_ECC_MUX_OFFS); reg_write(REG_DRAM_TRAINING_2_ADDR, reg); #ifdef RL_MODE reg_write(REG_DRAM_TRAINING_ADDR, 0); /* 0x15B0 - Training Register */ #endif /* Disable SW Read Leveling */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) & ~(1 << REG_DRAM_TRAINING_2_SW_OVRD_OFFS); /* [0] = 0 - Disable SW override */ reg = (reg | (0x1 << REG_DRAM_TRAINING_2_RL_MODE_OFFS)); /* [3] = 1 - Disable RL MODE */ /* 0x15B8 - Training SW 2 Register */ reg_write(REG_DRAM_TRAINING_2_ADDR, reg); DEBUG_RL_S("DDR3 - Read Leveling - Finished RL procedure for all CS\n"); return MV_OK; } #ifdef RL_MODE /* * overrun() extracted from ddr3_read_leveling_single_cs_rl_mode(). * This just got too much indented making it hard to read / edit. */ static void overrun(u32 cs, MV_DRAM_INFO *info, u32 pup, u32 locked_pups, u32 *locked_sum, u32 ecc, int *first_octet_locked, int *counter_in_progress, int final_delay, u32 delay, u32 phase) { /* If no OverRun */ if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) { int idx; idx = pup + ecc * ECC_BIT; /* PUP passed, start examining */ if (info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) { /* Must be RL_UNLOCK_STATE */ /* Match expected value ? - Update State Machine */ if (info->rl_val[cs][idx][C] < RL_RETRY_COUNT) { DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ", (u32)pup, 1); info->rl_val[cs][idx][C]++; /* If pup got to last state - lock the delays */ if (info->rl_val[cs][idx][C] == RL_RETRY_COUNT) { info->rl_val[cs][idx][C] = 0; info->rl_val[cs][idx][DS] = delay; info->rl_val[cs][idx][PS] = phase; /* Go to Final State */ info->rl_val[cs][idx][S] = RL_FINAL_STATE; *locked_sum = *locked_sum + 1; DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have locked pup: ", (u32)pup, 1); /* * If first lock - need to lock delays */ if (*first_octet_locked == 0) { DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ", (u32)pup, 1); *first_octet_locked = 1; } /* * If pup is in not in final state but * there was match - dont increment * counter */ } else { *counter_in_progress = 1; } } } } } /* * Name: ddr3_read_leveling_single_cs_rl_mode * Desc: Execute Read leveling for single Chip select * Args: cs - current chip select * freq - current sequence frequency * ecc - ecc iteration indication * dram_info - main struct * Notes: * Returns: MV_OK if success, MV_FAIL if fail. */ static int ddr3_read_leveling_single_cs_rl_mode(u32 cs, u32 freq, int ratio_2to1, u32 ecc, MV_DRAM_INFO *dram_info) { u32 reg, delay, phase, pup, rd_sample_delay, add, locked_pups, repeat_max_cnt, sdram_offset, locked_sum; u32 phase_min, ui_max_delay; int all_locked, first_octet_locked, counter_in_progress; int final_delay = 0; DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1); /* Init values */ phase = 0; delay = 0; rd_sample_delay = dram_info->cl; all_locked = 0; first_octet_locked = 0; repeat_max_cnt = 0; locked_sum = 0; for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0; /* Main loop */ while (!all_locked) { counter_in_progress = 0; DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = "); DEBUG_RL_FULL_D(rd_sample_delay, 2); DEBUG_RL_FULL_S(", RdRdyDly = "); DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2); DEBUG_RL_FULL_S(", Phase = "); DEBUG_RL_FULL_D(phase, 1); DEBUG_RL_FULL_S(", Delay = "); DEBUG_RL_FULL_D(delay, 2); DEBUG_RL_FULL_S("\n"); /* * Broadcast to all PUPs current RL delays: DQS phase, * leveling delay */ ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay); /* Reset PHY read FIFO */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); /* 0x15B8 - Training SW 2 Register */ reg_write(REG_DRAM_TRAINING_2_ADDR, reg); do { reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); } while (reg); /* Wait for '0' */ /* Read pattern from SDRAM */ sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS; locked_pups = 0; if (MV_OK != ddr3_sdram_compare(dram_info, 0xFF, &locked_pups, rl_pattern, LEN_STD_PATTERN, sdram_offset, 0, 0, NULL, 0)) return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PATTERN; /* Octet evaluation */ /* pup_num = Q or 1 for ECC */ for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { /* Check Overrun */ if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >> (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) { overrun(cs, dram_info, pup, locked_pups, &locked_sum, ecc, &first_octet_locked, &counter_in_progress, final_delay, delay, phase); } else { DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ", (u32)pup, 1); } } if (locked_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) { all_locked = 1; DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n"); } /* * This is a fix for unstable condition where pups are * toggling between match and no match */ /* * If some of the pups is >1 <3, check if we did it too * many times */ if (counter_in_progress == 1) { /* Notify at least one Counter is >=1 and < 3 */ if (repeat_max_cnt < RL_RETRY_COUNT) { repeat_max_cnt++; counter_in_progress = 1; DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n"); DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n"); } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n"); counter_in_progress = 0; } } /* * Check some of the pups are in the middle of state machine * and don't increment the delays */ if (!counter_in_progress && !all_locked) { int idx; idx = pup + ecc * ECC_BIT; repeat_max_cnt = 0; /* if 1:1 mode */ if ((!ratio_2to1) && ((phase == 0) || (phase == 4))) ui_max_delay = MAX_DELAY_INV; else ui_max_delay = MAX_DELAY; /* Increment Delay */ if (delay < ui_max_delay) { delay++; /* * Mark the last delay/pahse place for * window final place */ if (delay == ui_max_delay) { if ((!ratio_2to1 && phase == MAX_PHASE_RL_L_1TO1) || (ratio_2to1 && phase == MAX_PHASE_RL_L_2TO1)) final_delay = 1; } } else { /* Phase+CL Incrementation */ delay = 0; if (!ratio_2to1) { /* 1:1 mode */ if (first_octet_locked) { /* some Pup was Locked */ if (phase < MAX_PHASE_RL_L_1TO1) { if (phase == 1) { phase = 4; } else { phase++; delay = MIN_DELAY_PHASE_1_LIMIT; } } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); DEBUG_RL_S("1)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked n"); return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK; } } else { /* NO Pup was Locked */ if (phase < MAX_PHASE_RL_UL_1TO1) { phase++; delay = MIN_DELAY_PHASE_1_LIMIT; } else { phase = 0; } } } else { /* 2:1 mode */ if (first_octet_locked) { /* some Pup was Locked */ if (phase < MAX_PHASE_RL_L_2TO1) { phase++; } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); DEBUG_RL_S("2)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { /* pup_num = Q or 1 for ECC */ if (dram_info->rl_val[cs][idx][S] == 0) { DEBUG_RL_C("Failed byte is = ", pup, 1); } } return MV_DDR3_TRAINING_ERR_RD_LVL_RL_PUP_UNLOCK; } } else { /* No Pup was Locked */ if (phase < MAX_PHASE_RL_UL_2TO1) phase++; else phase = 0; } } /* * If we finished a full Phases cycle (so now * phase = 0, need to increment rd_sample_dly */ if (phase == 0 && first_octet_locked == 0) { rd_sample_delay++; if (rd_sample_delay == 0x10) { DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); DEBUG_RL_S("3)DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { /* pup_num = Q or 1 for ECC */ if (dram_info-> rl_val[cs][idx][S] == 0) { DEBUG_RL_C("Failed byte is = ", pup, 1); } } return MV_DDR3_TRAINING_ERR_RD_LVL_PUP_UNLOCK; } /* Set current rd_sample_delay */ reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg |= (rd_sample_delay << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg); } /* * Set current rdReadyDelay according to the * hash table (Need to do this in every phase * change) */ if (!ratio_2to1) { /* 1:1 mode */ add = reg_read(REG_TRAINING_DEBUG_2_ADDR); switch (phase) { case 0: add = (add >> REG_TRAINING_DEBUG_2_OFFS); break; case 1: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3)); break; case 4: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6)); break; case 5: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9)); break; } add &= REG_TRAINING_DEBUG_2_MASK; } else { /* 2:1 mode */ add = reg_read(REG_TRAINING_DEBUG_3_ADDR); add = (add >> (phase * REG_TRAINING_DEBUG_3_OFFS)); add &= REG_TRAINING_DEBUG_3_MASK; } reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); dram_info->rd_smpl_dly = rd_sample_delay; dram_info->rd_rdy_dly = rd_sample_delay + add; } /* Reset counters for pups with statesnum_of_std_pups * (1 - ecc) + ecc); pup++) { if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT) dram_info->rl_val[cs][idx][C] = 0; } } } phase_min = 10; for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) { if (dram_info->rl_val[cs][pup][PS] < phase_min) phase_min = dram_info->rl_val[cs][pup][PS]; } /* * Set current rdReadyDelay according to the hash table (Need to * do this in every phase change) */ if (!ratio_2to1) { /* 1:1 mode */ add = reg_read(REG_TRAINING_DEBUG_2_ADDR); switch (phase_min) { case 0: add = (add >> REG_TRAINING_DEBUG_2_OFFS); break; case 1: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3)); break; case 4: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6)); break; case 5: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9)); break; } add &= REG_TRAINING_DEBUG_2_MASK; } else { /* 2:1 mode */ add = reg_read(REG_TRAINING_DEBUG_3_ADDR); add = (add >> (phase_min * REG_TRAINING_DEBUG_3_OFFS)); add &= REG_TRAINING_DEBUG_3_MASK; } reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); dram_info->rd_rdy_dly = rd_sample_delay + add; for (cs = 0; cs < dram_info->num_cs; cs++) { for (pup = 0; pup < dram_info->num_of_total_pups; pup++) { reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup); dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F); } } return MV_OK; } #else /* * Name: ddr3_read_leveling_single_cs_window_mode * Desc: Execute Read leveling for single Chip select * Args: cs - current chip select * freq - current sequence frequency * ecc - ecc iteration indication * dram_info - main struct * Notes: * Returns: MV_OK if success, MV_FAIL if fail. */ static int ddr3_read_leveling_single_cs_window_mode(u32 cs, u32 freq, int ratio_2to1, u32 ecc, MV_DRAM_INFO *dram_info) { u32 reg, delay, phase, sum, pup, rd_sample_delay, add, locked_pups, repeat_max_cnt, sdram_offset, final_sum, locked_sum; u32 delay_s, delay_e, tmp, phase_min, ui_max_delay; int all_locked, first_octet_locked, counter_in_progress; int final_delay = 0; DEBUG_RL_FULL_C("DDR3 - Read Leveling - Single CS - ", (u32) cs, 1); /* Init values */ phase = 0; delay = 0; rd_sample_delay = dram_info->cl; all_locked = 0; first_octet_locked = 0; repeat_max_cnt = 0; sum = 0; final_sum = 0; locked_sum = 0; for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) dram_info->rl_val[cs][pup + ecc * ECC_BIT][S] = 0; /* Main loop */ while (!all_locked) { counter_in_progress = 0; DEBUG_RL_FULL_S("DDR3 - Read Leveling - RdSmplDly = "); DEBUG_RL_FULL_D(rd_sample_delay, 2); DEBUG_RL_FULL_S(", RdRdyDly = "); DEBUG_RL_FULL_D(dram_info->rd_rdy_dly, 2); DEBUG_RL_FULL_S(", Phase = "); DEBUG_RL_FULL_D(phase, 1); DEBUG_RL_FULL_S(", Delay = "); DEBUG_RL_FULL_D(delay, 2); DEBUG_RL_FULL_S("\n"); /* * Broadcast to all PUPs current RL delays: DQS phase,leveling * delay */ ddr3_write_pup_reg(PUP_RL_MODE, cs, PUP_BC, phase, delay); /* Reset PHY read FIFO */ reg = reg_read(REG_DRAM_TRAINING_2_ADDR) | (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); /* 0x15B8 - Training SW 2 Register */ reg_write(REG_DRAM_TRAINING_2_ADDR, reg); do { reg = (reg_read(REG_DRAM_TRAINING_2_ADDR)) & (1 << REG_DRAM_TRAINING_2_FIFO_RST_OFFS); } while (reg); /* Wait for '0' */ /* Read pattern from SDRAM */ sdram_offset = cs * (SDRAM_CS_SIZE + 1) + SDRAM_RL_OFFS; locked_pups = 0; if (MV_OK != ddr3_sdram_compare(dram_info, 0xFF, &locked_pups, rl_pattern, LEN_STD_PATTERN, sdram_offset, 0, 0, NULL, 0)) return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PATTERN; /* Octet evaluation */ for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { /* pup_num = Q or 1 for ECC */ int idx; idx = pup + ecc * ECC_BIT; /* Check Overrun */ if (!((reg_read(REG_DRAM_TRAINING_2_ADDR) >> (REG_DRAM_TRAINING_2_OVERRUN_OFFS + pup)) & 0x1)) { /* If no OverRun */ /* Inside the window */ if (dram_info->rl_val[cs][idx][S] == RL_WINDOW_STATE) { /* * Match expected value ? - Update * State Machine */ if (((~locked_pups >> pup) & 0x1) && (final_delay == 0)) { /* Match - Still inside the Window */ DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got another match inside the window for pup: ", (u32)pup, 1); } else { /* We got fail -> this is the end of the window */ dram_info->rl_val[cs][idx][DE] = delay; dram_info->rl_val[cs][idx][PE] = phase; /* Go to Final State */ dram_info->rl_val[cs][idx][S]++; final_sum++; DEBUG_RL_FULL_C("DDR3 - Read Leveling - We finished the window for pup: ", (u32)pup, 1); } /* Before the start of the window */ } else if (dram_info->rl_val[cs][idx][S] == RL_UNLOCK_STATE) { /* Must be RL_UNLOCK_STATE */ /* * Match expected value ? - Update * State Machine */ if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT) { if (((~locked_pups >> pup) & 0x1)) { /* Match */ DEBUG_RL_FULL_C("DDR3 - Read Leveling - We have no overrun and a match on pup: ", (u32)pup, 1); dram_info->rl_val[cs][idx][C]++; /* If pup got to last state - lock the delays */ if (dram_info->rl_val[cs][idx][C] == RL_RETRY_COUNT) { dram_info->rl_val[cs][idx][C] = 0; dram_info->rl_val[cs][idx][DS] = delay; dram_info->rl_val[cs][idx][PS] = phase; dram_info->rl_val[cs][idx][S]++; /* Go to Window State */ locked_sum++; /* Will count the pups that got locked */ /* IF First lock - need to lock delays */ if (first_octet_locked == 0) { DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got first lock on pup: ", (u32)pup, 1); first_octet_locked = 1; } } /* if pup is in not in final state but there was match - dont increment counter */ else { counter_in_progress = 1; } } } } } else { DEBUG_RL_FULL_C("DDR3 - Read Leveling - We got overrun on pup: ", (u32)pup, 1); counter_in_progress = 1; } } if (final_sum == (dram_info->num_of_std_pups * (1 - ecc) + ecc)) { all_locked = 1; DEBUG_RL_FULL_S("DDR3 - Read Leveling - Single Cs - All pups locked\n"); } /* * This is a fix for unstable condition where pups are * toggling between match and no match */ /* * If some of the pups is >1 <3, check if we did it too many * times */ if (counter_in_progress == 1) { if (repeat_max_cnt < RL_RETRY_COUNT) { /* Notify at least one Counter is >=1 and < 3 */ repeat_max_cnt++; counter_in_progress = 1; DEBUG_RL_FULL_S("DDR3 - Read Leveling - Counter is >=1 and <3\n"); DEBUG_RL_FULL_S("DDR3 - Read Leveling - So we will not increment the delay to see if locked again\n"); } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - repeat_max_cnt reached max so now we will increment the delay\n"); counter_in_progress = 0; } } /* * Check some of the pups are in the middle of state machine * and don't increment the delays */ if (!counter_in_progress && !all_locked) { repeat_max_cnt = 0; if (!ratio_2to1) ui_max_delay = MAX_DELAY_INV; else ui_max_delay = MAX_DELAY; /* Increment Delay */ if (delay < ui_max_delay) { /* Delay Incrementation */ delay++; if (delay == ui_max_delay) { /* * Mark the last delay/pahse place * for window final place */ if ((!ratio_2to1 && phase == MAX_PHASE_RL_L_1TO1) || (ratio_2to1 && phase == MAX_PHASE_RL_L_2TO1)) final_delay = 1; } } else { /* Phase+CL Incrementation */ delay = 0; if (!ratio_2to1) { /* 1:1 mode */ if (first_octet_locked) { /* some pupet was Locked */ if (phase < MAX_PHASE_RL_L_1TO1) { #ifdef RL_WINDOW_WA if (phase == 0) #else if (phase == 1) #endif phase = 4; else phase++; } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK; } } else { /* No Pup was Locked */ if (phase < MAX_PHASE_RL_UL_1TO1) { #ifdef RL_WINDOW_WA if (phase == 0) phase = 4; #else phase++; #endif } else phase = 0; } } else { /* 2:1 mode */ if (first_octet_locked) { /* Some Pup was Locked */ if (phase < MAX_PHASE_RL_L_2TO1) { phase++; } else { DEBUG_RL_FULL_S("DDR3 - Read Leveling - ERROR - NOT all PUPs Locked\n"); return MV_DDR3_TRAINING_ERR_RD_LVL_WIN_PUP_UNLOCK; } } else { /* No Pup was Locked */ if (phase < MAX_PHASE_RL_UL_2TO1) phase++; else phase = 0; } } /* * If we finished a full Phases cycle (so * now phase = 0, need to increment * rd_sample_dly */ if (phase == 0 && first_octet_locked == 0) { rd_sample_delay++; /* Set current rd_sample_delay */ reg = reg_read(REG_READ_DATA_SAMPLE_DELAYS_ADDR); reg &= ~(REG_READ_DATA_SAMPLE_DELAYS_MASK << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg |= (rd_sample_delay << (REG_READ_DATA_SAMPLE_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_SAMPLE_DELAYS_ADDR, reg); } /* * Set current rdReadyDelay according to the * hash table (Need to do this in every phase * change) */ if (!ratio_2to1) { /* 1:1 mode */ add = reg_read(REG_TRAINING_DEBUG_2_ADDR); switch (phase) { case 0: add = add >> REG_TRAINING_DEBUG_2_OFFS; break; case 1: add = add >> (REG_TRAINING_DEBUG_2_OFFS + 3); break; case 4: add = add >> (REG_TRAINING_DEBUG_2_OFFS + 6); break; case 5: add = add >> (REG_TRAINING_DEBUG_2_OFFS + 9); break; } } else { /* 2:1 mode */ add = reg_read(REG_TRAINING_DEBUG_3_ADDR); add = (add >> phase * REG_TRAINING_DEBUG_3_OFFS); } add &= REG_TRAINING_DEBUG_2_MASK; reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); dram_info->rd_smpl_dly = rd_sample_delay; dram_info->rd_rdy_dly = rd_sample_delay + add; } /* Reset counters for pups with statesnum_of_std_pups * (1 - ecc) + ecc); pup++) { if (dram_info->rl_val[cs][idx][C] < RL_RETRY_COUNT) dram_info->rl_val[cs][idx][C] = 0; } } } phase_min = 10; for (pup = 0; pup < (dram_info->num_of_std_pups); pup++) { DEBUG_RL_S("DDR3 - Read Leveling - Window info - PUP: "); DEBUG_RL_D((u32) pup, 1); DEBUG_RL_S(", PS: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PS], 1); DEBUG_RL_S(", DS: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DS], 2); DEBUG_RL_S(", PE: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][PE], 1); DEBUG_RL_S(", DE: "); DEBUG_RL_D((u32) dram_info->rl_val[cs][pup][DE], 2); DEBUG_RL_S("\n"); } /* Find center of the window procedure */ for (pup = 0; pup < (dram_info->num_of_std_pups * (1 - ecc) + ecc); pup++) { #ifdef RL_WINDOW_WA if (!ratio_2to1) { /* 1:1 mode */ if (dram_info->rl_val[cs][idx][PS] == 4) dram_info->rl_val[cs][idx][PS] = 1; if (dram_info->rl_val[cs][idx][PE] == 4) dram_info->rl_val[cs][idx][PE] = 1; delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY_INV + dram_info->rl_val[cs][idx][DS]; delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY_INV + dram_info->rl_val[cs][idx][DE]; tmp = (delay_e - delay_s) / 2 + delay_s; phase = tmp / MAX_DELAY_INV; if (phase == 1) /* 1:1 mode */ phase = 4; if (phase < phase_min) /* for the read ready delay */ phase_min = phase; dram_info->rl_val[cs][idx][P] = phase; dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY_INV; } else { delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY + dram_info->rl_val[cs][idx][DS]; delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY + dram_info->rl_val[cs][idx][DE]; tmp = (delay_e - delay_s) / 2 + delay_s; phase = tmp / MAX_DELAY; if (phase < phase_min) /* for the read ready delay */ phase_min = phase; dram_info->rl_val[cs][idx][P] = phase; dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY; } #else if (!ratio_2to1) { /* 1:1 mode */ if (dram_info->rl_val[cs][idx][PS] > 1) dram_info->rl_val[cs][idx][PS] -= 2; if (dram_info->rl_val[cs][idx][PE] > 1) dram_info->rl_val[cs][idx][PE] -= 2; } delay_s = dram_info->rl_val[cs][idx][PS] * MAX_DELAY + dram_info->rl_val[cs][idx][DS]; delay_e = dram_info->rl_val[cs][idx][PE] * MAX_DELAY + dram_info->rl_val[cs][idx][DE]; tmp = (delay_e - delay_s) / 2 + delay_s; phase = tmp / MAX_DELAY; if (!ratio_2to1 && phase > 1) /* 1:1 mode */ phase += 2; if (phase < phase_min) /* for the read ready delay */ phase_min = phase; dram_info->rl_val[cs][idx][P] = phase; dram_info->rl_val[cs][idx][D] = tmp % MAX_DELAY; #endif } /* Set current rdReadyDelay according to the hash table (Need to do this in every phase change) */ if (!ratio_2to1) { /* 1:1 mode */ add = reg_read(REG_TRAINING_DEBUG_2_ADDR); switch (phase_min) { case 0: add = (add >> REG_TRAINING_DEBUG_2_OFFS); break; case 1: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 3)); break; case 4: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 6)); break; case 5: add = (add >> (REG_TRAINING_DEBUG_2_OFFS + 9)); break; } } else { /* 2:1 mode */ add = reg_read(REG_TRAINING_DEBUG_3_ADDR); add = (add >> phase_min * REG_TRAINING_DEBUG_3_OFFS); } add &= REG_TRAINING_DEBUG_2_MASK; reg = reg_read(REG_READ_DATA_READY_DELAYS_ADDR); reg &= ~(REG_READ_DATA_READY_DELAYS_MASK << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg |= ((rd_sample_delay + add) << (REG_READ_DATA_READY_DELAYS_OFFS * cs)); reg_write(REG_READ_DATA_READY_DELAYS_ADDR, reg); dram_info->rd_rdy_dly = rd_sample_delay + add; for (cs = 0; cs < dram_info->num_cs; cs++) { for (pup = 0; pup < dram_info->num_of_total_pups; pup++) { reg = ddr3_read_pup_reg(PUP_RL_MODE + 0x1, cs, pup); dram_info->rl_val[cs][pup][DQS] = (reg & 0x3F); } } return MV_OK; } #endif