1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (c) 2016, The Linux Foundation. All rights reserved.
4	*/
5	#include <linux/bitops.h>
6	#include <linux/clk.h>
7	#include <linux/delay.h>
8	#include <linux/dmaengine.h>
9	#include <linux/dma-mapping.h>
10	#include <linux/dma/qcom_adm.h>
11	#include <linux/dma/qcom_bam_dma.h>
12	#include <linux/module.h>
13	#include <linux/mtd/partitions.h>
14	#include <linux/mtd/rawnand.h>
15	#include <linux/of.h>
16	#include <linux/platform_device.h>
17	#include <linux/slab.h>
18	#include <linux/mtd/nand-qpic-common.h>
19
20	/*
21	* NAND special boot partitions
22	*
23	* @page_offset: offset of the partition where spare data is not protected
24	* by ECC (value in pages)
25	* @page_offset: size of the partition where spare data is not protected
26	* by ECC (value in pages)
27	*/
28	struct qcom_nand_boot_partition {
29	u32 page_offset;
30	u32 page_size;
31	};
32
33	/*
34	* Qcom op for each exec_op transfer
35	*
36	* @data_instr: data instruction pointer
37	* @data_instr_idx: data instruction index
38	* @rdy_timeout_ms: wait ready timeout in ms
39	* @rdy_delay_ns: Additional delay in ns
40	* @addr1_reg: Address1 register value
41	* @addr2_reg: Address2 register value
42	* @cmd_reg: CMD register value
43	* @flag: flag for misc instruction
44	*/
45	struct qcom_op {
46	const struct nand_op_instr *data_instr;
47	unsigned int data_instr_idx;
48	unsigned int rdy_timeout_ms;
49	unsigned int rdy_delay_ns;
50	__le32 addr1_reg;
51	__le32 addr2_reg;
52	__le32 cmd_reg;
53	u8 flag;
54	};
55
56	/*
57	* NAND chip structure
58	*
59	* @boot_partitions: array of boot partitions where offset and size of the
60	* boot partitions are stored
61	*
62	* @chip: base NAND chip structure
63	* @node: list node to add itself to host_list in
64	* qcom_nand_controller
65	*
66	* @nr_boot_partitions: count of the boot partitions where spare data is not
67	* protected by ECC
68	*
69	* @cs: chip select value for this chip
70	* @cw_size: the number of bytes in a single step/codeword
71	* of a page, consisting of all data, ecc, spare
72	* and reserved bytes
73	* @cw_data: the number of bytes within a codeword protected
74	* by ECC
75	* @ecc_bytes_hw: ECC bytes used by controller hardware for this
76	* chip
77	*
78	* @last_command: keeps track of last command on this chip. used
79	* for reading correct status
80	*
81	* @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
82	* ecc/non-ecc mode for the current nand flash
83	* device
84	*
85	* @status: value to be returned if NAND_CMD_STATUS command
86	* is executed
87	* @codeword_fixup: keep track of the current layout used by
88	* the driver for read/write operation.
89	* @use_ecc: request the controller to use ECC for the
90	* upcoming read/write
91	* @bch_enabled: flag to tell whether BCH ECC mode is used
92	*/
93	struct qcom_nand_host {
94	struct qcom_nand_boot_partition *boot_partitions;
95
96	struct nand_chip chip;
97	struct list_head node;
98
99	int nr_boot_partitions;
100
101	int cs;
102	int cw_size;
103	int cw_data;
104	int ecc_bytes_hw;
105	int spare_bytes;
106	int bbm_size;
107
108	int last_command;
109
110	u32 cfg0, cfg1;
111	u32 cfg0_raw, cfg1_raw;
112	u32 ecc_buf_cfg;
113	u32 ecc_bch_cfg;
114	u32 clrflashstatus;
115	u32 clrreadstatus;
116
117	u8 status;
118	bool codeword_fixup;
119	bool use_ecc;
120	bool bch_enabled;
121	};
122
123	static struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
124	{
125	return container_of(chip, struct qcom_nand_host, chip);
126	}
127
128	static struct qcom_nand_controller *
129	get_qcom_nand_controller(struct nand_chip *chip)
130	{
131	return (struct qcom_nand_controller *)
132	((u8 *)chip->controller - sizeof(struct qcom_nand_controller));
133	}
134
135	static u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
136	{
137	return ioread32(nandc->base + offset);
138	}
139
140	static void nandc_write(struct qcom_nand_controller *nandc, int offset,
141	u32 val)
142	{
143	iowrite32(val, nandc->base + offset);
144	}
145
146	/* Helper to check whether this is the last CW or not */
147	static bool qcom_nandc_is_last_cw(struct nand_ecc_ctrl *ecc, int cw)
148	{
149	return cw == (ecc->steps - 1);
150	}
151
152	/**
153	* nandc_set_read_loc_first() - to set read location first register
154	* @chip: NAND Private Flash Chip Data
155	* @reg_base: location register base
156	* @cw_offset: code word offset
157	* @read_size: code word read length
158	* @is_last_read_loc: is this the last read location
159	*
160	* This function will set location register value
161	*/
162	static void nandc_set_read_loc_first(struct nand_chip *chip,
163	int reg_base, u32 cw_offset,
164	u32 read_size, u32 is_last_read_loc)
165	{
166	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
167	__le32 locreg_val;
168	u32 val = FIELD_PREP(READ_LOCATION_OFFSET_MASK, cw_offset) |
169	FIELD_PREP(READ_LOCATION_SIZE_MASK, read_size) |
170	FIELD_PREP(READ_LOCATION_LAST_MASK, is_last_read_loc);
171
172	locreg_val = cpu_to_le32(val);
173
174	if (reg_base == NAND_READ_LOCATION_0)
175	nandc->regs->read_location0 = locreg_val;
176	else if (reg_base == NAND_READ_LOCATION_1)
177	nandc->regs->read_location1 = locreg_val;
178	else if (reg_base == NAND_READ_LOCATION_2)
179	nandc->regs->read_location2 = locreg_val;
180	else if (reg_base == NAND_READ_LOCATION_3)
181	nandc->regs->read_location3 = locreg_val;
182	}
183
184	/**
185	* nandc_set_read_loc_last - to set read location last register
186	* @chip: NAND Private Flash Chip Data
187	* @reg_base: location register base
188	* @cw_offset: code word offset
189	* @read_size: code word read length
190	* @is_last_read_loc: is this the last read location
191	*
192	* This function will set location last register value
193	*/
194	static void nandc_set_read_loc_last(struct nand_chip *chip,
195	int reg_base, u32 cw_offset,
196	u32 read_size, u32 is_last_read_loc)
197	{
198	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
199	__le32 locreg_val;
200	u32 val = FIELD_PREP(READ_LOCATION_OFFSET_MASK, cw_offset) |
201	FIELD_PREP(READ_LOCATION_SIZE_MASK, read_size) |
202	FIELD_PREP(READ_LOCATION_LAST_MASK, is_last_read_loc);
203
204	locreg_val = cpu_to_le32(val);
205
206	if (reg_base == NAND_READ_LOCATION_LAST_CW_0)
207	nandc->regs->read_location_last0 = locreg_val;
208	else if (reg_base == NAND_READ_LOCATION_LAST_CW_1)
209	nandc->regs->read_location_last1 = locreg_val;
210	else if (reg_base == NAND_READ_LOCATION_LAST_CW_2)
211	nandc->regs->read_location_last2 = locreg_val;
212	else if (reg_base == NAND_READ_LOCATION_LAST_CW_3)
213	nandc->regs->read_location_last3 = locreg_val;
214	}
215
216	/* helper to configure location register values */
217	static void nandc_set_read_loc(struct nand_chip *chip, int cw, int reg,
218	u32 cw_offset, u32 read_size, u32 is_last_read_loc)
219	{
220	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
221	struct nand_ecc_ctrl *ecc = &chip->ecc;
222	int reg_base = NAND_READ_LOCATION_0;
223
224	if (nandc->props->qpic_version2 && qcom_nandc_is_last_cw(ecc, cw))
225	reg_base = NAND_READ_LOCATION_LAST_CW_0;
226
227	reg_base += reg * 4;
228
229	if (nandc->props->qpic_version2 && qcom_nandc_is_last_cw(ecc, cw))
230	return nandc_set_read_loc_last(chip, reg_base, cw_offset,
231	read_size, is_last_read_loc);
232	else
233	return nandc_set_read_loc_first(chip, reg_base, cw_offset,
234	read_size, is_last_read_loc);
235	}
236
237	/* helper to configure address register values */
238	static void set_address(struct qcom_nand_host *host, u16 column, int page)
239	{
240	struct nand_chip *chip = &host->chip;
241	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
242
243	if (chip->options & NAND_BUSWIDTH_16)
244	column >>= 1;
245
246	nandc->regs->addr0 = cpu_to_le32(page << 16 | column);
247	nandc->regs->addr1 = cpu_to_le32(page >> 16 & 0xff);
248	}
249
250	/*
251	* update_rw_regs: set up read/write register values, these will be
252	* written to the NAND controller registers via DMA
253	*
254	* @num_cw: number of steps for the read/write operation
255	* @read: read or write operation
256	* @cw : which code word
257	*/
258	static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read, int cw)
259	{
260	struct nand_chip *chip = &host->chip;
261	__le32 cmd, cfg0, cfg1, ecc_bch_cfg;
262	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
263
264	if (read) {
265	if (host->use_ecc)
266	cmd = cpu_to_le32(OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE);
267	else
268	cmd = cpu_to_le32(OP_PAGE_READ | PAGE_ACC | LAST_PAGE);
269	} else {
270	cmd = cpu_to_le32(OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE);
271	}
272
273	if (host->use_ecc) {
274	cfg0 = cpu_to_le32((host->cfg0 & ~CW_PER_PAGE_MASK) |
275	FIELD_PREP(CW_PER_PAGE_MASK, (num_cw - 1)));
276
277	cfg1 = cpu_to_le32(host->cfg1);
278	ecc_bch_cfg = cpu_to_le32(host->ecc_bch_cfg);
279	} else {
280	cfg0 = cpu_to_le32((host->cfg0_raw & ~CW_PER_PAGE_MASK) |
281	FIELD_PREP(CW_PER_PAGE_MASK, (num_cw - 1)));
282
283	cfg1 = cpu_to_le32(host->cfg1_raw);
284	ecc_bch_cfg = cpu_to_le32(ECC_CFG_ECC_DISABLE);
285	}
286
287	nandc->regs->cmd = cmd;
288	nandc->regs->cfg0 = cfg0;
289	nandc->regs->cfg1 = cfg1;
290	nandc->regs->ecc_bch_cfg = ecc_bch_cfg;
291
292	if (!nandc->props->qpic_version2)
293	nandc->regs->ecc_buf_cfg = cpu_to_le32(host->ecc_buf_cfg);
294
295	nandc->regs->clrflashstatus = cpu_to_le32(host->clrflashstatus);
296	nandc->regs->clrreadstatus = cpu_to_le32(host->clrreadstatus);
297	nandc->regs->exec = cpu_to_le32(1);
298
299	if (read)
300	nandc_set_read_loc(chip, cw, reg: 0, cw_offset: 0, read_size: host->use_ecc ?
301	host->cw_data : host->cw_size, is_last_read_loc: 1);
302	}
303
304	/*
305	* Helper to prepare DMA descriptors for configuring registers
306	* before reading a NAND page.
307	*/
308	static void config_nand_page_read(struct nand_chip *chip)
309	{
310	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
311
312	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->addr0, NAND_ADDR0, num_regs: 2, flags: 0);
313	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cfg0, NAND_DEV0_CFG0, num_regs: 3, flags: 0);
314	if (!nandc->props->qpic_version2)
315	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG, num_regs: 1, flags: 0);
316	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->erased_cw_detect_cfg_clr,
317	NAND_ERASED_CW_DETECT_CFG, num_regs: 1, flags: 0);
318	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->erased_cw_detect_cfg_set,
319	NAND_ERASED_CW_DETECT_CFG, num_regs: 1, NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
320	}
321
322	/*
323	* Helper to prepare DMA descriptors for configuring registers
324	* before reading each codeword in NAND page.
325	*/
326	static void
327	config_nand_cw_read(struct nand_chip *chip, bool use_ecc, int cw)
328	{
329	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
330	struct nand_ecc_ctrl *ecc = &chip->ecc;
331
332	__le32 *reg = &nandc->regs->read_location0;
333
334	if (nandc->props->qpic_version2 && qcom_nandc_is_last_cw(ecc, cw))
335	reg = &nandc->regs->read_location_last0;
336
337	if (nandc->props->supports_bam)
338	qcom_write_reg_dma(nandc, vaddr: reg, NAND_READ_LOCATION_0, num_regs: 4, NAND_BAM_NEXT_SGL);
339
340	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd, NAND_FLASH_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
341	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->exec, NAND_EXEC_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
342
343	if (use_ecc) {
344	qcom_read_reg_dma(nandc, NAND_FLASH_STATUS, num_regs: 2, flags: 0);
345	qcom_read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, num_regs: 1,
346	NAND_BAM_NEXT_SGL);
347	} else {
348	qcom_read_reg_dma(nandc, NAND_FLASH_STATUS, num_regs: 1, NAND_BAM_NEXT_SGL);
349	}
350	}
351
352	/*
353	* Helper to prepare dma descriptors to configure registers needed for reading a
354	* single codeword in page
355	*/
356	static void
357	config_nand_single_cw_page_read(struct nand_chip *chip,
358	bool use_ecc, int cw)
359	{
360	config_nand_page_read(chip);
361	config_nand_cw_read(chip, use_ecc, cw);
362	}
363
364	/*
365	* Helper to prepare DMA descriptors used to configure registers needed for
366	* before writing a NAND page.
367	*/
368	static void config_nand_page_write(struct nand_chip *chip)
369	{
370	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
371
372	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->addr0, NAND_ADDR0, num_regs: 2, flags: 0);
373	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cfg0, NAND_DEV0_CFG0, num_regs: 3, flags: 0);
374	if (!nandc->props->qpic_version2)
375	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->ecc_buf_cfg, NAND_EBI2_ECC_BUF_CFG, num_regs: 1,
376	NAND_BAM_NEXT_SGL);
377	}
378
379	/*
380	* Helper to prepare DMA descriptors for configuring registers
381	* before writing each codeword in NAND page.
382	*/
383	static void config_nand_cw_write(struct nand_chip *chip)
384	{
385	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
386
387	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd, NAND_FLASH_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
388	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->exec, NAND_EXEC_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
389
390	qcom_read_reg_dma(nandc, NAND_FLASH_STATUS, num_regs: 1, NAND_BAM_NEXT_SGL);
391
392	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->clrflashstatus, NAND_FLASH_STATUS, num_regs: 1, flags: 0);
393	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->clrreadstatus, NAND_READ_STATUS, num_regs: 1,
394	NAND_BAM_NEXT_SGL);
395	}
396
397	/*
398	* when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
399	* an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
400	*
401	* when using RS ECC, the HW reports the same erros when reading an erased CW,
402	* but it notifies that it is an erased CW by placing special characters at
403	* certain offsets in the buffer.
404	*
405	* verify if the page is erased or not, and fix up the page for RS ECC by
406	* replacing the special characters with 0xff.
407	*/
408	static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
409	{
410	u8 empty1, empty2;
411
412	/*
413	* an erased page flags an error in NAND_FLASH_STATUS, check if the page
414	* is erased by looking for 0x54s at offsets 3 and 175 from the
415	* beginning of each codeword
416	*/
417
418	empty1 = data_buf[3];
419	empty2 = data_buf[175];
420
421	/*
422	* if the erased codework markers, if they exist override them with
423	* 0xffs
424	*/
425	if ((empty1 == 0x54 && empty2 == 0xff) ||
426	(empty1 == 0xff && empty2 == 0x54)) {
427	data_buf[3] = 0xff;
428	data_buf[175] = 0xff;
429	}
430
431	/*
432	* check if the entire chunk contains 0xffs or not. if it doesn't, then
433	* restore the original values at the special offsets
434	*/
435	if (memchr_inv(p: data_buf, c: 0xff, size: data_len)) {
436	data_buf[3] = empty1;
437	data_buf[175] = empty2;
438
439	return false;
440	}
441
442	return true;
443	}
444
445	struct read_stats {
446	__le32 flash;
447	__le32 buffer;
448	__le32 erased_cw;
449	};
450
451	/* reads back FLASH_STATUS register set by the controller */
452	static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
453	{
454	struct nand_chip *chip = &host->chip;
455	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
456	int i;
457
458	qcom_nandc_dev_to_mem(nandc, is_cpu: true);
459
460	for (i = 0; i < cw_cnt; i++) {
461	u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
462
463	if (flash & (FS_OP_ERR | FS_MPU_ERR))
464	return -EIO;
465	}
466
467	return 0;
468	}
469
470	/* performs raw read for one codeword */
471	static int
472	qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
473	u8 *data_buf, u8 *oob_buf, int page, int cw)
474	{
475	struct qcom_nand_host *host = to_qcom_nand_host(chip);
476	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
477	struct nand_ecc_ctrl *ecc = &chip->ecc;
478	int data_size1, data_size2, oob_size1, oob_size2;
479	int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
480	int raw_cw = cw;
481
482	nand_read_page_op(chip, page, offset_in_page: 0, NULL, len: 0);
483	nandc->buf_count = 0;
484	nandc->buf_start = 0;
485	qcom_clear_read_regs(nandc);
486	host->use_ecc = false;
487
488	if (nandc->props->qpic_version2)
489	raw_cw = ecc->steps - 1;
490
491	qcom_clear_bam_transaction(nandc);
492	set_address(host, column: host->cw_size * cw, page);
493	update_rw_regs(host, num_cw: 1, read: true, cw: raw_cw);
494	config_nand_page_read(chip);
495
496	data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
497	oob_size1 = host->bbm_size;
498
499	if (qcom_nandc_is_last_cw(ecc, cw) && !host->codeword_fixup) {
500	data_size2 = ecc->size - data_size1 -
501	((ecc->steps - 1) * 4);
502	oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
503	host->spare_bytes;
504	} else {
505	data_size2 = host->cw_data - data_size1;
506	oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
507	}
508
509	if (nandc->props->supports_bam) {
510	nandc_set_read_loc(chip, cw, reg: 0, cw_offset: read_loc, read_size: data_size1, is_last_read_loc: 0);
511	read_loc += data_size1;
512
513	nandc_set_read_loc(chip, cw, reg: 1, cw_offset: read_loc, read_size: oob_size1, is_last_read_loc: 0);
514	read_loc += oob_size1;
515
516	nandc_set_read_loc(chip, cw, reg: 2, cw_offset: read_loc, read_size: data_size2, is_last_read_loc: 0);
517	read_loc += data_size2;
518
519	nandc_set_read_loc(chip, cw, reg: 3, cw_offset: read_loc, read_size: oob_size2, is_last_read_loc: 1);
520	}
521
522	config_nand_cw_read(chip, use_ecc: false, cw: raw_cw);
523
524	qcom_read_data_dma(nandc, reg_off, vaddr: data_buf, size: data_size1, flags: 0);
525	reg_off += data_size1;
526
527	qcom_read_data_dma(nandc, reg_off, vaddr: oob_buf, size: oob_size1, flags: 0);
528	reg_off += oob_size1;
529
530	qcom_read_data_dma(nandc, reg_off, vaddr: data_buf + data_size1, size: data_size2, flags: 0);
531	reg_off += data_size2;
532
533	qcom_read_data_dma(nandc, reg_off, vaddr: oob_buf + oob_size1, size: oob_size2, flags: 0);
534
535	ret = qcom_submit_descs(nandc);
536	if (ret) {
537	dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
538	return ret;
539	}
540
541	return check_flash_errors(host, cw_cnt: 1);
542	}
543
544	/*
545	* Bitflips can happen in erased codewords also so this function counts the
546	* number of 0 in each CW for which ECC engine returns the uncorrectable
547	* error. The page will be assumed as erased if this count is less than or
548	* equal to the ecc->strength for each CW.
549	*
550	* 1. Both DATA and OOB need to be checked for number of 0. The
551	* top-level API can be called with only data buf or OOB buf so use
552	* chip->data_buf if data buf is null and chip->oob_poi if oob buf
553	* is null for copying the raw bytes.
554	* 2. Perform raw read for all the CW which has uncorrectable errors.
555	* 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
556	* The BBM and spare bytes bit flip won’t affect the ECC so don’t check
557	* the number of bitflips in this area.
558	*/
559	static int
560	check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
561	u8 *oob_buf, unsigned long uncorrectable_cws,
562	int page, unsigned int max_bitflips)
563	{
564	struct nand_chip *chip = &host->chip;
565	struct mtd_info *mtd = nand_to_mtd(chip);
566	struct nand_ecc_ctrl *ecc = &chip->ecc;
567	u8 *cw_data_buf, *cw_oob_buf;
568	int cw, data_size, oob_size, ret;
569
570	if (!data_buf)
571	data_buf = nand_get_data_buf(chip);
572
573	if (!oob_buf) {
574	nand_get_data_buf(chip);
575	oob_buf = chip->oob_poi;
576	}
577
578	for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
579	if (qcom_nandc_is_last_cw(ecc, cw) && !host->codeword_fixup) {
580	data_size = ecc->size - ((ecc->steps - 1) * 4);
581	oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
582	} else {
583	data_size = host->cw_data;
584	oob_size = host->ecc_bytes_hw;
585	}
586
587	/* determine starting buffer address for current CW */
588	cw_data_buf = data_buf + (cw * host->cw_data);
589	cw_oob_buf = oob_buf + (cw * ecc->bytes);
590
591	ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf: cw_data_buf,
592	oob_buf: cw_oob_buf, page, cw);
593	if (ret)
594	return ret;
595
596	/*
597	* make sure it isn't an erased page reported
598	* as not-erased by HW because of a few bitflips
599	*/
600	ret = nand_check_erased_ecc_chunk(data: cw_data_buf, datalen: data_size,
601	ecc: cw_oob_buf + host->bbm_size,
602	ecclen: oob_size, NULL,
603	extraooblen: 0, threshold: ecc->strength);
604	if (ret < 0) {
605	mtd->ecc_stats.failed++;
606	} else {
607	mtd->ecc_stats.corrected += ret;
608	max_bitflips = max_t(unsigned int, max_bitflips, ret);
609	}
610	}
611
612	return max_bitflips;
613	}
614
615	/*
616	* reads back status registers set by the controller to notify page read
617	* errors. this is equivalent to what 'ecc->correct()' would do.
618	*/
619	static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
620	u8 *oob_buf, int page)
621	{
622	struct nand_chip *chip = &host->chip;
623	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
624	struct mtd_info *mtd = nand_to_mtd(chip);
625	struct nand_ecc_ctrl *ecc = &chip->ecc;
626	unsigned int max_bitflips = 0, uncorrectable_cws = 0;
627	struct read_stats *buf;
628	bool flash_op_err = false, erased;
629	int i;
630	u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
631
632	buf = (struct read_stats *)nandc->reg_read_buf;
633	qcom_nandc_dev_to_mem(nandc, is_cpu: true);
634
635	for (i = 0; i < ecc->steps; i++, buf++) {
636	u32 flash, buffer, erased_cw;
637	int data_len, oob_len;
638
639	if (qcom_nandc_is_last_cw(ecc, cw: i)) {
640	data_len = ecc->size - ((ecc->steps - 1) << 2);
641	oob_len = ecc->steps << 2;
642	} else {
643	data_len = host->cw_data;
644	oob_len = 0;
645	}
646
647	flash = le32_to_cpu(buf->flash);
648	buffer = le32_to_cpu(buf->buffer);
649	erased_cw = le32_to_cpu(buf->erased_cw);
650
651	/*
652	* Check ECC failure for each codeword. ECC failure can
653	* happen in either of the following conditions
654	* 1. If number of bitflips are greater than ECC engine
655	* capability.
656	* 2. If this codeword contains all 0xff for which erased
657	* codeword detection check will be done.
658	*/
659	if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
660	/*
661	* For BCH ECC, ignore erased codeword errors, if
662	* ERASED_CW bits are set.
663	*/
664	if (host->bch_enabled) {
665	erased = (erased_cw & ERASED_CW) == ERASED_CW;
666	/*
667	* For RS ECC, HW reports the erased CW by placing
668	* special characters at certain offsets in the buffer.
669	* These special characters will be valid only if
670	* complete page is read i.e. data_buf is not NULL.
671	*/
672	} else if (data_buf) {
673	erased = erased_chunk_check_and_fixup(data_buf,
674	data_len);
675	} else {
676	erased = false;
677	}
678
679	if (!erased)
680	uncorrectable_cws |= BIT(i);
681	/*
682	* Check if MPU or any other operational error (timeout,
683	* device failure, etc.) happened for this codeword and
684	* make flash_op_err true. If flash_op_err is set, then
685	* EIO will be returned for page read.
686	*/
687	} else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
688	flash_op_err = true;
689	/*
690	* No ECC or operational errors happened. Check the number of
691	* bits corrected and update the ecc_stats.corrected.
692	*/
693	} else {
694	unsigned int stat;
695
696	stat = buffer & BS_CORRECTABLE_ERR_MSK;
697	mtd->ecc_stats.corrected += stat;
698	max_bitflips = max(max_bitflips, stat);
699	}
700
701	if (data_buf)
702	data_buf += data_len;
703	if (oob_buf)
704	oob_buf += oob_len + ecc->bytes;
705	}
706
707	if (flash_op_err)
708	return -EIO;
709
710	if (!uncorrectable_cws)
711	return max_bitflips;
712
713	return check_for_erased_page(host, data_buf: data_buf_start, oob_buf: oob_buf_start,
714	uncorrectable_cws, page,
715	max_bitflips);
716	}
717
718	/*
719	* helper to perform the actual page read operation, used by ecc->read_page(),
720	* ecc->read_oob()
721	*/
722	static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
723	u8 *oob_buf, int page)
724	{
725	struct nand_chip *chip = &host->chip;
726	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
727	struct nand_ecc_ctrl *ecc = &chip->ecc;
728	u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
729	int i, ret;
730
731	config_nand_page_read(chip);
732
733	/* queue cmd descs for each codeword */
734	for (i = 0; i < ecc->steps; i++) {
735	int data_size, oob_size;
736
737	if (qcom_nandc_is_last_cw(ecc, cw: i) && !host->codeword_fixup) {
738	data_size = ecc->size - ((ecc->steps - 1) << 2);
739	oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
740	host->spare_bytes;
741	} else {
742	data_size = host->cw_data;
743	oob_size = host->ecc_bytes_hw + host->spare_bytes;
744	}
745
746	if (nandc->props->supports_bam) {
747	if (data_buf && oob_buf) {
748	nandc_set_read_loc(chip, cw: i, reg: 0, cw_offset: 0, read_size: data_size, is_last_read_loc: 0);
749	nandc_set_read_loc(chip, cw: i, reg: 1, cw_offset: data_size,
750	read_size: oob_size, is_last_read_loc: 1);
751	} else if (data_buf) {
752	nandc_set_read_loc(chip, cw: i, reg: 0, cw_offset: 0, read_size: data_size, is_last_read_loc: 1);
753	} else {
754	nandc_set_read_loc(chip, cw: i, reg: 0, cw_offset: data_size,
755	read_size: oob_size, is_last_read_loc: 1);
756	}
757	}
758
759	config_nand_cw_read(chip, use_ecc: true, cw: i);
760
761	if (data_buf)
762	qcom_read_data_dma(nandc, FLASH_BUF_ACC, vaddr: data_buf,
763	size: data_size, flags: 0);
764
765	/*
766	* when ecc is enabled, the controller doesn't read the real
767	* or dummy bad block markers in each chunk. To maintain a
768	* consistent layout across RAW and ECC reads, we just
769	* leave the real/dummy BBM offsets empty (i.e, filled with
770	* 0xffs)
771	*/
772	if (oob_buf) {
773	int j;
774
775	for (j = 0; j < host->bbm_size; j++)
776	*oob_buf++ = 0xff;
777
778	qcom_read_data_dma(nandc, FLASH_BUF_ACC + data_size,
779	vaddr: oob_buf, size: oob_size, flags: 0);
780	}
781
782	if (data_buf)
783	data_buf += data_size;
784	if (oob_buf)
785	oob_buf += oob_size;
786	}
787
788	ret = qcom_submit_descs(nandc);
789	if (ret) {
790	dev_err(nandc->dev, "failure to read page/oob\n");
791	return ret;
792	}
793
794	return parse_read_errors(host, data_buf: data_buf_start, oob_buf: oob_buf_start, page);
795	}
796
797	/*
798	* a helper that copies the last step/codeword of a page (containing free oob)
799	* into our local buffer
800	*/
801	static int copy_last_cw(struct qcom_nand_host *host, int page)
802	{
803	struct nand_chip *chip = &host->chip;
804	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
805	struct nand_ecc_ctrl *ecc = &chip->ecc;
806	int size;
807	int ret;
808
809	qcom_clear_read_regs(nandc);
810
811	size = host->use_ecc ? host->cw_data : host->cw_size;
812
813	/* prepare a clean read buffer */
814	memset(nandc->data_buffer, 0xff, size);
815
816	set_address(host, column: host->cw_size * (ecc->steps - 1), page);
817	update_rw_regs(host, num_cw: 1, read: true, cw: ecc->steps - 1);
818
819	config_nand_single_cw_page_read(chip, use_ecc: host->use_ecc, cw: ecc->steps - 1);
820
821	qcom_read_data_dma(nandc, FLASH_BUF_ACC, vaddr: nandc->data_buffer, size, flags: 0);
822
823	ret = qcom_submit_descs(nandc);
824	if (ret)
825	dev_err(nandc->dev, "failed to copy last codeword\n");
826
827	return ret;
828	}
829
830	static bool qcom_nandc_is_boot_partition(struct qcom_nand_host *host, int page)
831	{
832	struct qcom_nand_boot_partition *boot_partition;
833	u32 start, end;
834	int i;
835
836	/*
837	* Since the frequent access will be to the non-boot partitions like rootfs,
838	* optimize the page check by:
839	*
840	* 1. Checking if the page lies after the last boot partition.
841	* 2. Checking from the boot partition end.
842	*/
843
844	/* First check the last boot partition */
845	boot_partition = &host->boot_partitions[host->nr_boot_partitions - 1];
846	start = boot_partition->page_offset;
847	end = start + boot_partition->page_size;
848
849	/* Page is after the last boot partition end. This is NOT a boot partition */
850	if (page > end)
851	return false;
852
853	/* Actually check if it's a boot partition */
854	if (page < end && page >= start)
855	return true;
856
857	/* Check the other boot partitions starting from the second-last partition */
858	for (i = host->nr_boot_partitions - 2; i >= 0; i--) {
859	boot_partition = &host->boot_partitions[i];
860	start = boot_partition->page_offset;
861	end = start + boot_partition->page_size;
862
863	if (page < end && page >= start)
864	return true;
865	}
866
867	return false;
868	}
869
870	static void qcom_nandc_codeword_fixup(struct qcom_nand_host *host, int page)
871	{
872	bool codeword_fixup = qcom_nandc_is_boot_partition(host, page);
873
874	/* Skip conf write if we are already in the correct mode */
875	if (codeword_fixup == host->codeword_fixup)
876	return;
877
878	host->codeword_fixup = codeword_fixup;
879
880	host->cw_data = codeword_fixup ? 512 : 516;
881	host->spare_bytes = host->cw_size - host->ecc_bytes_hw -
882	host->bbm_size - host->cw_data;
883
884	host->cfg0 &= ~(SPARE_SIZE_BYTES_MASK | UD_SIZE_BYTES_MASK);
885	host->cfg0 |= FIELD_PREP(SPARE_SIZE_BYTES_MASK, host->spare_bytes) |
886	FIELD_PREP(UD_SIZE_BYTES_MASK, host->cw_data);
887
888	host->ecc_bch_cfg &= ~ECC_NUM_DATA_BYTES_MASK;
889	host->ecc_bch_cfg |= FIELD_PREP(ECC_NUM_DATA_BYTES_MASK, host->cw_data);
890	host->ecc_buf_cfg = FIELD_PREP(NUM_STEPS_MASK, host->cw_data - 1);
891	}
892
893	/* implements ecc->read_page() */
894	static int qcom_nandc_read_page(struct nand_chip *chip, u8 *buf,
895	int oob_required, int page)
896	{
897	struct qcom_nand_host *host = to_qcom_nand_host(chip);
898	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
899	struct nand_ecc_ctrl *ecc = &chip->ecc;
900	u8 *data_buf, *oob_buf = NULL;
901
902	if (host->nr_boot_partitions)
903	qcom_nandc_codeword_fixup(host, page);
904
905	nand_read_page_op(chip, page, offset_in_page: 0, NULL, len: 0);
906	nandc->buf_count = 0;
907	nandc->buf_start = 0;
908	host->use_ecc = true;
909	qcom_clear_read_regs(nandc);
910	set_address(host, column: 0, page);
911	update_rw_regs(host, num_cw: ecc->steps, read: true, cw: 0);
912
913	data_buf = buf;
914	oob_buf = oob_required ? chip->oob_poi : NULL;
915
916	qcom_clear_bam_transaction(nandc);
917
918	return read_page_ecc(host, data_buf, oob_buf, page);
919	}
920
921	/* implements ecc->read_page_raw() */
922	static int qcom_nandc_read_page_raw(struct nand_chip *chip, u8 *buf,
923	int oob_required, int page)
924	{
925	struct mtd_info *mtd = nand_to_mtd(chip);
926	struct qcom_nand_host *host = to_qcom_nand_host(chip);
927	struct nand_ecc_ctrl *ecc = &chip->ecc;
928	int cw, ret;
929	u8 *data_buf = buf, *oob_buf = chip->oob_poi;
930
931	if (host->nr_boot_partitions)
932	qcom_nandc_codeword_fixup(host, page);
933
934	for (cw = 0; cw < ecc->steps; cw++) {
935	ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
936	page, cw);
937	if (ret)
938	return ret;
939
940	data_buf += host->cw_data;
941	oob_buf += ecc->bytes;
942	}
943
944	return 0;
945	}
946
947	/* implements ecc->read_oob() */
948	static int qcom_nandc_read_oob(struct nand_chip *chip, int page)
949	{
950	struct qcom_nand_host *host = to_qcom_nand_host(chip);
951	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
952	struct nand_ecc_ctrl *ecc = &chip->ecc;
953
954	if (host->nr_boot_partitions)
955	qcom_nandc_codeword_fixup(host, page);
956
957	qcom_clear_read_regs(nandc);
958	qcom_clear_bam_transaction(nandc);
959
960	host->use_ecc = true;
961	set_address(host, column: 0, page);
962	update_rw_regs(host, num_cw: ecc->steps, read: true, cw: 0);
963
964	return read_page_ecc(host, NULL, oob_buf: chip->oob_poi, page);
965	}
966
967	/* implements ecc->write_page() */
968	static int qcom_nandc_write_page(struct nand_chip *chip, const u8 *buf,
969	int oob_required, int page)
970	{
971	struct qcom_nand_host *host = to_qcom_nand_host(chip);
972	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
973	struct nand_ecc_ctrl *ecc = &chip->ecc;
974	u8 *data_buf, *oob_buf;
975	int i, ret;
976
977	if (host->nr_boot_partitions)
978	qcom_nandc_codeword_fixup(host, page);
979
980	nand_prog_page_begin_op(chip, page, offset_in_page: 0, NULL, len: 0);
981
982	set_address(host, column: 0, page);
983	nandc->buf_count = 0;
984	nandc->buf_start = 0;
985	qcom_clear_read_regs(nandc);
986	qcom_clear_bam_transaction(nandc);
987
988	data_buf = (u8 *)buf;
989	oob_buf = chip->oob_poi;
990
991	host->use_ecc = true;
992	update_rw_regs(host, num_cw: ecc->steps, read: false, cw: 0);
993	config_nand_page_write(chip);
994
995	for (i = 0; i < ecc->steps; i++) {
996	int data_size, oob_size;
997
998	if (qcom_nandc_is_last_cw(ecc, cw: i) && !host->codeword_fixup) {
999	data_size = ecc->size - ((ecc->steps - 1) << 2);
1000	oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1001	host->spare_bytes;
1002	} else {
1003	data_size = host->cw_data;
1004	oob_size = ecc->bytes;
1005	}
1006
1007	qcom_write_data_dma(nandc, FLASH_BUF_ACC, vaddr: data_buf, size: data_size,
1008	flags: i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
1009
1010	/*
1011	* when ECC is enabled, we don't really need to write anything
1012	* to oob for the first n - 1 codewords since these oob regions
1013	* just contain ECC bytes that's written by the controller
1014	* itself. For the last codeword, we skip the bbm positions and
1015	* write to the free oob area.
1016	*/
1017	if (qcom_nandc_is_last_cw(ecc, cw: i)) {
1018	oob_buf += host->bbm_size;
1019
1020	qcom_write_data_dma(nandc, FLASH_BUF_ACC + data_size,
1021	vaddr: oob_buf, size: oob_size, flags: 0);
1022	}
1023
1024	config_nand_cw_write(chip);
1025
1026	data_buf += data_size;
1027	oob_buf += oob_size;
1028	}
1029
1030	ret = qcom_submit_descs(nandc);
1031	if (ret) {
1032	dev_err(nandc->dev, "failure to write page\n");
1033	return ret;
1034	}
1035
1036	return nand_prog_page_end_op(chip);
1037	}
1038
1039	/* implements ecc->write_page_raw() */
1040	static int qcom_nandc_write_page_raw(struct nand_chip *chip,
1041	const u8 *buf, int oob_required,
1042	int page)
1043	{
1044	struct mtd_info *mtd = nand_to_mtd(chip);
1045	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1046	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1047	struct nand_ecc_ctrl *ecc = &chip->ecc;
1048	u8 *data_buf, *oob_buf;
1049	int i, ret;
1050
1051	if (host->nr_boot_partitions)
1052	qcom_nandc_codeword_fixup(host, page);
1053
1054	nand_prog_page_begin_op(chip, page, offset_in_page: 0, NULL, len: 0);
1055	qcom_clear_read_regs(nandc);
1056	qcom_clear_bam_transaction(nandc);
1057
1058	data_buf = (u8 *)buf;
1059	oob_buf = chip->oob_poi;
1060
1061	host->use_ecc = false;
1062	update_rw_regs(host, num_cw: ecc->steps, read: false, cw: 0);
1063	config_nand_page_write(chip);
1064
1065	for (i = 0; i < ecc->steps; i++) {
1066	int data_size1, data_size2, oob_size1, oob_size2;
1067	int reg_off = FLASH_BUF_ACC;
1068
1069	data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1070	oob_size1 = host->bbm_size;
1071
1072	if (qcom_nandc_is_last_cw(ecc, cw: i) && !host->codeword_fixup) {
1073	data_size2 = ecc->size - data_size1 -
1074	((ecc->steps - 1) << 2);
1075	oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
1076	host->spare_bytes;
1077	} else {
1078	data_size2 = host->cw_data - data_size1;
1079	oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1080	}
1081
1082	qcom_write_data_dma(nandc, reg_off, vaddr: data_buf, size: data_size1,
1083	NAND_BAM_NO_EOT);
1084	reg_off += data_size1;
1085	data_buf += data_size1;
1086
1087	qcom_write_data_dma(nandc, reg_off, vaddr: oob_buf, size: oob_size1,
1088	NAND_BAM_NO_EOT);
1089	reg_off += oob_size1;
1090	oob_buf += oob_size1;
1091
1092	qcom_write_data_dma(nandc, reg_off, vaddr: data_buf, size: data_size2,
1093	NAND_BAM_NO_EOT);
1094	reg_off += data_size2;
1095	data_buf += data_size2;
1096
1097	qcom_write_data_dma(nandc, reg_off, vaddr: oob_buf, size: oob_size2, flags: 0);
1098	oob_buf += oob_size2;
1099
1100	config_nand_cw_write(chip);
1101	}
1102
1103	ret = qcom_submit_descs(nandc);
1104	if (ret) {
1105	dev_err(nandc->dev, "failure to write raw page\n");
1106	return ret;
1107	}
1108
1109	return nand_prog_page_end_op(chip);
1110	}
1111
1112	/*
1113	* implements ecc->write_oob()
1114	*
1115	* the NAND controller cannot write only data or only OOB within a codeword
1116	* since ECC is calculated for the combined codeword. So update the OOB from
1117	* chip->oob_poi, and pad the data area with OxFF before writing.
1118	*/
1119	static int qcom_nandc_write_oob(struct nand_chip *chip, int page)
1120	{
1121	struct mtd_info *mtd = nand_to_mtd(chip);
1122	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1123	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1124	struct nand_ecc_ctrl *ecc = &chip->ecc;
1125	u8 *oob = chip->oob_poi;
1126	int data_size, oob_size;
1127	int ret;
1128
1129	if (host->nr_boot_partitions)
1130	qcom_nandc_codeword_fixup(host, page);
1131
1132	host->use_ecc = true;
1133	qcom_clear_bam_transaction(nandc);
1134
1135	/* calculate the data and oob size for the last codeword/step */
1136	data_size = ecc->size - ((ecc->steps - 1) << 2);
1137	oob_size = mtd->oobavail;
1138
1139	memset(nandc->data_buffer, 0xff, host->cw_data);
1140	/* override new oob content to last codeword */
1141	mtd_ooblayout_get_databytes(mtd, databuf: nandc->data_buffer + data_size, oobbuf: oob,
1142	start: 0, nbytes: mtd->oobavail);
1143
1144	set_address(host, column: host->cw_size * (ecc->steps - 1), page);
1145	update_rw_regs(host, num_cw: 1, read: false, cw: 0);
1146
1147	config_nand_page_write(chip);
1148	qcom_write_data_dma(nandc, FLASH_BUF_ACC,
1149	vaddr: nandc->data_buffer, size: data_size + oob_size, flags: 0);
1150	config_nand_cw_write(chip);
1151
1152	ret = qcom_submit_descs(nandc);
1153	if (ret) {
1154	dev_err(nandc->dev, "failure to write oob\n");
1155	return ret;
1156	}
1157
1158	return nand_prog_page_end_op(chip);
1159	}
1160
1161	static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs)
1162	{
1163	struct mtd_info *mtd = nand_to_mtd(chip);
1164	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1165	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1166	struct nand_ecc_ctrl *ecc = &chip->ecc;
1167	int page, ret, bbpos, bad = 0;
1168
1169	page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1170
1171	/*
1172	* configure registers for a raw sub page read, the address is set to
1173	* the beginning of the last codeword, we don't care about reading ecc
1174	* portion of oob. we just want the first few bytes from this codeword
1175	* that contains the BBM
1176	*/
1177	host->use_ecc = false;
1178
1179	qcom_clear_bam_transaction(nandc);
1180	ret = copy_last_cw(host, page);
1181	if (ret)
1182	goto err;
1183
1184	if (check_flash_errors(host, cw_cnt: 1)) {
1185	dev_warn(nandc->dev, "error when trying to read BBM\n");
1186	goto err;
1187	}
1188
1189	bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
1190
1191	bad = nandc->data_buffer[bbpos] != 0xff;
1192
1193	if (chip->options & NAND_BUSWIDTH_16)
1194	bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
1195	err:
1196	return bad;
1197	}
1198
1199	static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs)
1200	{
1201	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1202	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1203	struct nand_ecc_ctrl *ecc = &chip->ecc;
1204	int page, ret;
1205
1206	qcom_clear_read_regs(nandc);
1207	qcom_clear_bam_transaction(nandc);
1208
1209	/*
1210	* to mark the BBM as bad, we flash the entire last codeword with 0s.
1211	* we don't care about the rest of the content in the codeword since
1212	* we aren't going to use this block again
1213	*/
1214	memset(nandc->data_buffer, 0x00, host->cw_size);
1215
1216	page = (int)(ofs >> chip->page_shift) & chip->pagemask;
1217
1218	/* prepare write */
1219	host->use_ecc = false;
1220	set_address(host, column: host->cw_size * (ecc->steps - 1), page);
1221	update_rw_regs(host, num_cw: 1, read: false, cw: ecc->steps - 1);
1222
1223	config_nand_page_write(chip);
1224	qcom_write_data_dma(nandc, FLASH_BUF_ACC,
1225	vaddr: nandc->data_buffer, size: host->cw_size, flags: 0);
1226	config_nand_cw_write(chip);
1227
1228	ret = qcom_submit_descs(nandc);
1229	if (ret) {
1230	dev_err(nandc->dev, "failure to update BBM\n");
1231	return ret;
1232	}
1233
1234	return nand_prog_page_end_op(chip);
1235	}
1236
1237	/*
1238	* NAND controller page layout info
1239	*
1240	* Layout with ECC enabled:
1241	*
1242	* |----------------------| |---------------------------------|
1243	* | xx.......yy| | *********xx.......yy|
1244	* | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
1245	* | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
1246	* | xx.......yy| | *********xx.......yy|
1247	* |----------------------| |---------------------------------|
1248	* codeword 1,2..n-1 codeword n
1249	* <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
1250	*
1251	* n = Number of codewords in the page
1252	* . = ECC bytes
1253	* * = Spare/free bytes
1254	* x = Unused byte(s)
1255	* y = Reserved byte(s)
1256	*
1257	* 2K page: n = 4, spare = 16 bytes
1258	* 4K page: n = 8, spare = 32 bytes
1259	* 8K page: n = 16, spare = 64 bytes
1260	*
1261	* the qcom nand controller operates at a sub page/codeword level. each
1262	* codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
1263	* the number of ECC bytes vary based on the ECC strength and the bus width.
1264	*
1265	* the first n - 1 codewords contains 516 bytes of user data, the remaining
1266	* 12/16 bytes consist of ECC and reserved data. The nth codeword contains
1267	* both user data and spare(oobavail) bytes that sum up to 516 bytes.
1268	*
1269	* When we access a page with ECC enabled, the reserved bytes(s) are not
1270	* accessible at all. When reading, we fill up these unreadable positions
1271	* with 0xffs. When writing, the controller skips writing the inaccessible
1272	* bytes.
1273	*
1274	* Layout with ECC disabled:
1275	*
1276	* |------------------------------| |---------------------------------------|
1277	* | yy xx.......| | bb *********xx.......|
1278	* | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
1279	* | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
1280	* | yy xx.......| | bb *********xx.......|
1281	* |------------------------------| |---------------------------------------|
1282	* codeword 1,2..n-1 codeword n
1283	* <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
1284	*
1285	* n = Number of codewords in the page
1286	* . = ECC bytes
1287	* * = Spare/free bytes
1288	* x = Unused byte(s)
1289	* y = Dummy Bad Bock byte(s)
1290	* b = Real Bad Block byte(s)
1291	* size1/size2 = function of codeword size and 'n'
1292	*
1293	* when the ECC block is disabled, one reserved byte (or two for 16 bit bus
1294	* width) is now accessible. For the first n - 1 codewords, these are dummy Bad
1295	* Block Markers. In the last codeword, this position contains the real BBM
1296	*
1297	* In order to have a consistent layout between RAW and ECC modes, we assume
1298	* the following OOB layout arrangement:
1299	*
1300	* |-----------| |--------------------|
1301	* |yyxx.......| |bb*********xx.......|
1302	* |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
1303	* |yyxx.......| |bb*********xx.......|
1304	* |yyxx.......| |bb*********xx.......|
1305	* |-----------| |--------------------|
1306	* first n - 1 nth OOB region
1307	* OOB regions
1308	*
1309	* n = Number of codewords in the page
1310	* . = ECC bytes
1311	* * = FREE OOB bytes
1312	* y = Dummy bad block byte(s) (inaccessible when ECC enabled)
1313	* x = Unused byte(s)
1314	* b = Real bad block byte(s) (inaccessible when ECC enabled)
1315	*
1316	* This layout is read as is when ECC is disabled. When ECC is enabled, the
1317	* inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
1318	* and assumed as 0xffs when we read a page/oob. The ECC, unused and
1319	* dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
1320	* the sum of the three).
1321	*/
1322	static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1323	struct mtd_oob_region *oobregion)
1324	{
1325	struct nand_chip *chip = mtd_to_nand(mtd);
1326	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1327	struct nand_ecc_ctrl *ecc = &chip->ecc;
1328
1329	if (section > 1)
1330	return -ERANGE;
1331
1332	if (!section) {
1333	oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
1334	host->bbm_size;
1335	oobregion->offset = 0;
1336	} else {
1337	oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
1338	oobregion->offset = mtd->oobsize - oobregion->length;
1339	}
1340
1341	return 0;
1342	}
1343
1344	static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
1345	struct mtd_oob_region *oobregion)
1346	{
1347	struct nand_chip *chip = mtd_to_nand(mtd);
1348	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1349	struct nand_ecc_ctrl *ecc = &chip->ecc;
1350
1351	if (section)
1352	return -ERANGE;
1353
1354	oobregion->length = ecc->steps * 4;
1355	oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
1356
1357	return 0;
1358	}
1359
1360	static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
1361	.ecc = qcom_nand_ooblayout_ecc,
1362	.free = qcom_nand_ooblayout_free,
1363	};
1364
1365	static int
1366	qcom_nandc_calc_ecc_bytes(int step_size, int strength)
1367	{
1368	return strength == 4 ? 12 : 16;
1369	}
1370
1371	NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
1372	NANDC_STEP_SIZE, 4, 8);
1373
1374	static int qcom_nand_attach_chip(struct nand_chip *chip)
1375	{
1376	struct mtd_info *mtd = nand_to_mtd(chip);
1377	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1378	struct nand_ecc_ctrl *ecc = &chip->ecc;
1379	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1380	int cwperpage, bad_block_byte, ret;
1381	bool wide_bus;
1382	int ecc_mode = 1;
1383
1384	/* controller only supports 512 bytes data steps */
1385	ecc->size = NANDC_STEP_SIZE;
1386	wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
1387	cwperpage = mtd->writesize / NANDC_STEP_SIZE;
1388
1389	/*
1390	* Each CW has 4 available OOB bytes which will be protected with ECC
1391	* so remaining bytes can be used for ECC.
1392	*/
1393	ret = nand_ecc_choose_conf(chip, caps: &qcom_nandc_ecc_caps,
1394	oobavail: mtd->oobsize - (cwperpage * 4));
1395	if (ret) {
1396	dev_err(nandc->dev, "No valid ECC settings possible\n");
1397	return ret;
1398	}
1399
1400	if (ecc->strength >= 8) {
1401	/* 8 bit ECC defaults to BCH ECC on all platforms */
1402	host->bch_enabled = true;
1403	ecc_mode = 1;
1404
1405	if (wide_bus) {
1406	host->ecc_bytes_hw = 14;
1407	host->spare_bytes = 0;
1408	host->bbm_size = 2;
1409	} else {
1410	host->ecc_bytes_hw = 13;
1411	host->spare_bytes = 2;
1412	host->bbm_size = 1;
1413	}
1414	} else {
1415	/*
1416	* if the controller supports BCH for 4 bit ECC, the controller
1417	* uses lesser bytes for ECC. If RS is used, the ECC bytes is
1418	* always 10 bytes
1419	*/
1420	if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
1421	/* BCH */
1422	host->bch_enabled = true;
1423	ecc_mode = 0;
1424
1425	if (wide_bus) {
1426	host->ecc_bytes_hw = 8;
1427	host->spare_bytes = 2;
1428	host->bbm_size = 2;
1429	} else {
1430	host->ecc_bytes_hw = 7;
1431	host->spare_bytes = 4;
1432	host->bbm_size = 1;
1433	}
1434	} else {
1435	/* RS */
1436	host->ecc_bytes_hw = 10;
1437
1438	if (wide_bus) {
1439	host->spare_bytes = 0;
1440	host->bbm_size = 2;
1441	} else {
1442	host->spare_bytes = 1;
1443	host->bbm_size = 1;
1444	}
1445	}
1446	}
1447
1448	/*
1449	* we consider ecc->bytes as the sum of all the non-data content in a
1450	* step. It gives us a clean representation of the oob area (even if
1451	* all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
1452	* ECC and 12 bytes for 4 bit ECC
1453	*/
1454	ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
1455
1456	ecc->read_page = qcom_nandc_read_page;
1457	ecc->read_page_raw = qcom_nandc_read_page_raw;
1458	ecc->read_oob = qcom_nandc_read_oob;
1459	ecc->write_page = qcom_nandc_write_page;
1460	ecc->write_page_raw = qcom_nandc_write_page_raw;
1461	ecc->write_oob = qcom_nandc_write_oob;
1462
1463	ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1464
1465	mtd_set_ooblayout(mtd, ooblayout: &qcom_nand_ooblayout_ops);
1466	/* Free the initially allocated BAM transaction for reading the ONFI params */
1467	if (nandc->props->supports_bam)
1468	qcom_free_bam_transaction(nandc);
1469
1470	nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
1471	cwperpage);
1472
1473	/* Now allocate the BAM transaction based on updated max_cwperpage */
1474	if (nandc->props->supports_bam) {
1475	nandc->bam_txn = qcom_alloc_bam_transaction(nandc);
1476	if (!nandc->bam_txn) {
1477	dev_err(nandc->dev,
1478	"failed to allocate bam transaction\n");
1479	return -ENOMEM;
1480	}
1481	}
1482
1483	/*
1484	* DATA_UD_BYTES varies based on whether the read/write command protects
1485	* spare data with ECC too. We protect spare data by default, so we set
1486	* it to main + spare data, which are 512 and 4 bytes respectively.
1487	*/
1488	host->cw_data = 516;
1489
1490	/*
1491	* total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
1492	* for 8 bit ECC
1493	*/
1494	host->cw_size = host->cw_data + ecc->bytes;
1495	bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
1496
1497	host->cfg0 = FIELD_PREP(CW_PER_PAGE_MASK, (cwperpage - 1)) |
1498	FIELD_PREP(UD_SIZE_BYTES_MASK, host->cw_data) |
1499	FIELD_PREP(DISABLE_STATUS_AFTER_WRITE, 0) |
1500	FIELD_PREP(NUM_ADDR_CYCLES_MASK, 5) |
1501	FIELD_PREP(ECC_PARITY_SIZE_BYTES_RS, host->ecc_bytes_hw) |
1502	FIELD_PREP(STATUS_BFR_READ, 0) |
1503	FIELD_PREP(SET_RD_MODE_AFTER_STATUS, 1) |
1504	FIELD_PREP(SPARE_SIZE_BYTES_MASK, host->spare_bytes);
1505
1506	host->cfg1 = FIELD_PREP(NAND_RECOVERY_CYCLES_MASK, 7) |
1507	FIELD_PREP(BAD_BLOCK_BYTE_NUM_MASK, bad_block_byte) |
1508	FIELD_PREP(BAD_BLOCK_IN_SPARE_AREA, 0) |
1509	FIELD_PREP(WR_RD_BSY_GAP_MASK, 2) |
1510	FIELD_PREP(WIDE_FLASH, wide_bus) |
1511	FIELD_PREP(ENABLE_BCH_ECC, host->bch_enabled);
1512
1513	host->cfg0_raw = FIELD_PREP(CW_PER_PAGE_MASK, (cwperpage - 1)) |
1514	FIELD_PREP(UD_SIZE_BYTES_MASK, host->cw_size) |
1515	FIELD_PREP(NUM_ADDR_CYCLES_MASK, 5) |
1516	FIELD_PREP(SPARE_SIZE_BYTES_MASK, 0);
1517
1518	host->cfg1_raw = FIELD_PREP(NAND_RECOVERY_CYCLES_MASK, 7) |
1519	FIELD_PREP(CS_ACTIVE_BSY, 0) |
1520	FIELD_PREP(BAD_BLOCK_BYTE_NUM_MASK, 17) |
1521	FIELD_PREP(BAD_BLOCK_IN_SPARE_AREA, 1) |
1522	FIELD_PREP(WR_RD_BSY_GAP_MASK, 2) |
1523	FIELD_PREP(WIDE_FLASH, wide_bus) |
1524	FIELD_PREP(DEV0_CFG1_ECC_DISABLE, 1);
1525
1526	host->ecc_bch_cfg = FIELD_PREP(ECC_CFG_ECC_DISABLE, !host->bch_enabled) |
1527	FIELD_PREP(ECC_SW_RESET, 0) |
1528	FIELD_PREP(ECC_NUM_DATA_BYTES_MASK, host->cw_data) |
1529	FIELD_PREP(ECC_FORCE_CLK_OPEN, 1) |
1530	FIELD_PREP(ECC_MODE_MASK, ecc_mode) |
1531	FIELD_PREP(ECC_PARITY_SIZE_BYTES_BCH_MASK, host->ecc_bytes_hw);
1532
1533	if (!nandc->props->qpic_version2)
1534	host->ecc_buf_cfg = FIELD_PREP(NUM_STEPS_MASK, 0x203);
1535
1536	host->clrflashstatus = FS_READY_BSY_N;
1537	host->clrreadstatus = 0xc0;
1538	nandc->regs->erased_cw_detect_cfg_clr =
1539	cpu_to_le32(CLR_ERASED_PAGE_DET);
1540	nandc->regs->erased_cw_detect_cfg_set =
1541	cpu_to_le32(SET_ERASED_PAGE_DET);
1542
1543	dev_dbg(nandc->dev,
1544	"cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
1545	host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
1546	host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
1547	cwperpage);
1548
1549	return 0;
1550	}
1551
1552	static int qcom_op_cmd_mapping(struct nand_chip *chip, u8 opcode,
1553	struct qcom_op *q_op)
1554	{
1555	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1556	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1557	int cmd;
1558
1559	switch (opcode) {
1560	case NAND_CMD_RESET:
1561	cmd = OP_RESET_DEVICE;
1562	break;
1563	case NAND_CMD_READID:
1564	cmd = OP_FETCH_ID;
1565	break;
1566	case NAND_CMD_PARAM:
1567	if (nandc->props->qpic_version2)
1568	cmd = OP_PAGE_READ_ONFI_READ;
1569	else
1570	cmd = OP_PAGE_READ;
1571	break;
1572	case NAND_CMD_ERASE1:
1573	case NAND_CMD_ERASE2:
1574	cmd = OP_BLOCK_ERASE;
1575	break;
1576	case NAND_CMD_STATUS:
1577	cmd = OP_CHECK_STATUS;
1578	break;
1579	case NAND_CMD_PAGEPROG:
1580	cmd = OP_PROGRAM_PAGE;
1581	q_op->flag = OP_PROGRAM_PAGE;
1582	nandc->exec_opwrite = true;
1583	break;
1584	case NAND_CMD_READ0:
1585	case NAND_CMD_READSTART:
1586	if (host->use_ecc)
1587	cmd = OP_PAGE_READ_WITH_ECC;
1588	else
1589	cmd = OP_PAGE_READ;
1590	break;
1591	default:
1592	dev_err(nandc->dev, "Opcode not supported: %u\n", opcode);
1593	return -EOPNOTSUPP;
1594	}
1595
1596	return cmd;
1597	}
1598
1599	/* NAND framework ->exec_op() hooks and related helpers */
1600	static int qcom_parse_instructions(struct nand_chip *chip,
1601	const struct nand_subop *subop,
1602	struct qcom_op *q_op)
1603	{
1604	const struct nand_op_instr *instr = NULL;
1605	unsigned int op_id;
1606	int i, ret;
1607
1608	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1609	unsigned int offset, naddrs;
1610	const u8 *addrs;
1611
1612	instr = &subop->instrs[op_id];
1613
1614	switch (instr->type) {
1615	case NAND_OP_CMD_INSTR:
1616	ret = qcom_op_cmd_mapping(chip, opcode: instr->ctx.cmd.opcode, q_op);
1617	if (ret < 0)
1618	return ret;
1619
1620	q_op->cmd_reg = cpu_to_le32(ret);
1621	q_op->rdy_delay_ns = instr->delay_ns;
1622	break;
1623
1624	case NAND_OP_ADDR_INSTR:
1625	offset = nand_subop_get_addr_start_off(subop, op_id);
1626	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1627	addrs = &instr->ctx.addr.addrs[offset];
1628
1629	for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1630	q_op->addr1_reg |= cpu_to_le32(addrs[i] << (i * 8));
1631
1632	if (naddrs > 4)
1633	q_op->addr2_reg |= cpu_to_le32(addrs[4]);
1634
1635	q_op->rdy_delay_ns = instr->delay_ns;
1636	break;
1637
1638	case NAND_OP_DATA_IN_INSTR:
1639	q_op->data_instr = instr;
1640	q_op->data_instr_idx = op_id;
1641	q_op->rdy_delay_ns = instr->delay_ns;
1642	fallthrough;
1643	case NAND_OP_DATA_OUT_INSTR:
1644	q_op->rdy_delay_ns = instr->delay_ns;
1645	break;
1646
1647	case NAND_OP_WAITRDY_INSTR:
1648	q_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1649	q_op->rdy_delay_ns = instr->delay_ns;
1650	break;
1651	}
1652	}
1653
1654	return 0;
1655	}
1656
1657	static void qcom_delay_ns(unsigned int ns)
1658	{
1659	if (!ns)
1660	return;
1661
1662	if (ns < 10000)
1663	ndelay(ns);
1664	else
1665	udelay(DIV_ROUND_UP(ns, 1000));
1666	}
1667
1668	static int qcom_wait_rdy_poll(struct nand_chip *chip, unsigned int time_ms)
1669	{
1670	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1671	unsigned long start = jiffies + msecs_to_jiffies(m: time_ms);
1672	u32 flash;
1673
1674	qcom_nandc_dev_to_mem(nandc, is_cpu: true);
1675
1676	do {
1677	flash = le32_to_cpu(nandc->reg_read_buf[0]);
1678	if (flash & FS_READY_BSY_N)
1679	return 0;
1680	cpu_relax();
1681	} while (time_after(start, jiffies));
1682
1683	dev_err(nandc->dev, "Timeout waiting for device to be ready:0x%08x\n", flash);
1684
1685	return -ETIMEDOUT;
1686	}
1687
1688	static int qcom_read_status_exec(struct nand_chip *chip,
1689	const struct nand_subop *subop)
1690	{
1691	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1692	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1693	struct nand_ecc_ctrl *ecc = &chip->ecc;
1694	struct qcom_op q_op = {};
1695	const struct nand_op_instr *instr = NULL;
1696	unsigned int op_id = 0;
1697	unsigned int len = 0;
1698	int ret, num_cw, i;
1699	u32 flash_status;
1700
1701	host->status = NAND_STATUS_READY | NAND_STATUS_WP;
1702
1703	ret = qcom_parse_instructions(chip, subop, q_op: &q_op);
1704	if (ret)
1705	return ret;
1706
1707	num_cw = nandc->exec_opwrite ? ecc->steps : 1;
1708	nandc->exec_opwrite = false;
1709
1710	nandc->buf_count = 0;
1711	nandc->buf_start = 0;
1712	host->use_ecc = false;
1713
1714	qcom_clear_read_regs(nandc);
1715	qcom_clear_bam_transaction(nandc);
1716
1717	nandc->regs->cmd = q_op.cmd_reg;
1718	nandc->regs->exec = cpu_to_le32(1);
1719
1720	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd, NAND_FLASH_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
1721	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->exec, NAND_EXEC_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
1722	qcom_read_reg_dma(nandc, NAND_FLASH_STATUS, num_regs: 1, NAND_BAM_NEXT_SGL);
1723
1724	ret = qcom_submit_descs(nandc);
1725	if (ret) {
1726	dev_err(nandc->dev, "failure in submitting status descriptor\n");
1727	goto err_out;
1728	}
1729
1730	qcom_nandc_dev_to_mem(nandc, is_cpu: true);
1731
1732	for (i = 0; i < num_cw; i++) {
1733	flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1734
1735	if (flash_status & FS_MPU_ERR)
1736	host->status &= ~NAND_STATUS_WP;
1737
1738	if (flash_status & FS_OP_ERR ||
1739	(i == (num_cw - 1) && (flash_status & FS_DEVICE_STS_ERR)))
1740	host->status |= NAND_STATUS_FAIL;
1741	}
1742
1743	flash_status = host->status;
1744	instr = q_op.data_instr;
1745	op_id = q_op.data_instr_idx;
1746	len = nand_subop_get_data_len(subop, op_id);
1747	memcpy(instr->ctx.data.buf.in, &flash_status, len);
1748
1749	err_out:
1750	return ret;
1751	}
1752
1753	static int qcom_read_id_type_exec(struct nand_chip *chip, const struct nand_subop *subop)
1754	{
1755	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1756	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1757	struct qcom_op q_op = {};
1758	const struct nand_op_instr *instr = NULL;
1759	unsigned int op_id = 0;
1760	unsigned int len = 0;
1761	int ret;
1762
1763	ret = qcom_parse_instructions(chip, subop, q_op: &q_op);
1764	if (ret)
1765	return ret;
1766
1767	nandc->buf_count = 0;
1768	nandc->buf_start = 0;
1769	host->use_ecc = false;
1770
1771	qcom_clear_read_regs(nandc);
1772	qcom_clear_bam_transaction(nandc);
1773
1774	nandc->regs->cmd = q_op.cmd_reg;
1775	nandc->regs->addr0 = q_op.addr1_reg;
1776	nandc->regs->addr1 = q_op.addr2_reg;
1777	nandc->regs->chip_sel = cpu_to_le32(nandc->props->supports_bam ? 0 : DM_EN);
1778	nandc->regs->exec = cpu_to_le32(1);
1779
1780	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd, NAND_FLASH_CMD, num_regs: 4, NAND_BAM_NEXT_SGL);
1781	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->exec, NAND_EXEC_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
1782
1783	qcom_read_reg_dma(nandc, NAND_READ_ID, num_regs: 1, NAND_BAM_NEXT_SGL);
1784
1785	ret = qcom_submit_descs(nandc);
1786	if (ret) {
1787	dev_err(nandc->dev, "failure in submitting read id descriptor\n");
1788	goto err_out;
1789	}
1790
1791	instr = q_op.data_instr;
1792	op_id = q_op.data_instr_idx;
1793	len = nand_subop_get_data_len(subop, op_id);
1794
1795	qcom_nandc_dev_to_mem(nandc, is_cpu: true);
1796	memcpy(instr->ctx.data.buf.in, nandc->reg_read_buf, len);
1797
1798	err_out:
1799	return ret;
1800	}
1801
1802	static int qcom_misc_cmd_type_exec(struct nand_chip *chip, const struct nand_subop *subop)
1803	{
1804	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1805	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1806	struct qcom_op q_op = {};
1807	int ret;
1808	int instrs = 1;
1809
1810	ret = qcom_parse_instructions(chip, subop, q_op: &q_op);
1811	if (ret)
1812	return ret;
1813
1814	if (q_op.flag == OP_PROGRAM_PAGE) {
1815	goto wait_rdy;
1816	} else if (q_op.cmd_reg == cpu_to_le32(OP_BLOCK_ERASE)) {
1817	q_op.cmd_reg |= cpu_to_le32(PAGE_ACC | LAST_PAGE);
1818	nandc->regs->addr0 = q_op.addr1_reg;
1819	nandc->regs->addr1 = q_op.addr2_reg;
1820	nandc->regs->cfg0 = cpu_to_le32(host->cfg0_raw & ~CW_PER_PAGE_MASK);
1821	nandc->regs->cfg1 = cpu_to_le32(host->cfg1_raw);
1822	instrs = 3;
1823	} else if (q_op.cmd_reg != cpu_to_le32(OP_RESET_DEVICE)) {
1824	return 0;
1825	}
1826
1827	nandc->buf_count = 0;
1828	nandc->buf_start = 0;
1829	host->use_ecc = false;
1830
1831	qcom_clear_read_regs(nandc);
1832	qcom_clear_bam_transaction(nandc);
1833
1834	nandc->regs->cmd = q_op.cmd_reg;
1835	nandc->regs->exec = cpu_to_le32(1);
1836
1837	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd, NAND_FLASH_CMD, num_regs: instrs, NAND_BAM_NEXT_SGL);
1838	if (q_op.cmd_reg == cpu_to_le32(OP_BLOCK_ERASE))
1839	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cfg0, NAND_DEV0_CFG0, num_regs: 2, NAND_BAM_NEXT_SGL);
1840
1841	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->exec, NAND_EXEC_CMD, num_regs: 1, NAND_BAM_NEXT_SGL);
1842	qcom_read_reg_dma(nandc, NAND_FLASH_STATUS, num_regs: 1, NAND_BAM_NEXT_SGL);
1843
1844	ret = qcom_submit_descs(nandc);
1845	if (ret) {
1846	dev_err(nandc->dev, "failure in submitting misc descriptor\n");
1847	goto err_out;
1848	}
1849
1850	wait_rdy:
1851	qcom_delay_ns(ns: q_op.rdy_delay_ns);
1852	ret = qcom_wait_rdy_poll(chip, time_ms: q_op.rdy_timeout_ms);
1853
1854	err_out:
1855	return ret;
1856	}
1857
1858	static int qcom_param_page_type_exec(struct nand_chip *chip, const struct nand_subop *subop)
1859	{
1860	struct qcom_nand_host *host = to_qcom_nand_host(chip);
1861	struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1862	struct qcom_op q_op = {};
1863	const struct nand_op_instr *instr = NULL;
1864	unsigned int op_id = 0;
1865	unsigned int len = 0;
1866	int ret, reg_base;
1867
1868	reg_base = NAND_READ_LOCATION_0;
1869
1870	if (nandc->props->qpic_version2)
1871	reg_base = NAND_READ_LOCATION_LAST_CW_0;
1872
1873	ret = qcom_parse_instructions(chip, subop, q_op: &q_op);
1874	if (ret)
1875	return ret;
1876
1877	q_op.cmd_reg |= cpu_to_le32(PAGE_ACC | LAST_PAGE);
1878
1879	nandc->buf_count = 0;
1880	nandc->buf_start = 0;
1881	host->use_ecc = false;
1882	qcom_clear_read_regs(nandc);
1883	qcom_clear_bam_transaction(nandc);
1884
1885	nandc->regs->cmd = q_op.cmd_reg;
1886	nandc->regs->addr0 = 0;
1887	nandc->regs->addr1 = 0;
1888
1889	nandc->regs->cfg0 = cpu_to_le32(FIELD_PREP(CW_PER_PAGE_MASK, 0) |
1890	FIELD_PREP(UD_SIZE_BYTES_MASK, 512) |
1891	FIELD_PREP(NUM_ADDR_CYCLES_MASK, 5) |
1892	FIELD_PREP(SPARE_SIZE_BYTES_MASK, 0));
1893
1894	nandc->regs->cfg1 = cpu_to_le32(FIELD_PREP(NAND_RECOVERY_CYCLES_MASK, 7) |
1895	FIELD_PREP(BAD_BLOCK_BYTE_NUM_MASK, 17) |
1896	FIELD_PREP(CS_ACTIVE_BSY, 0) |
1897	FIELD_PREP(BAD_BLOCK_IN_SPARE_AREA, 1) |
1898	FIELD_PREP(WR_RD_BSY_GAP_MASK, 2) |
1899	FIELD_PREP(WIDE_FLASH, 0) |
1900	FIELD_PREP(DEV0_CFG1_ECC_DISABLE, 1));
1901
1902	if (!nandc->props->qpic_version2)
1903	nandc->regs->ecc_buf_cfg = cpu_to_le32(ECC_CFG_ECC_DISABLE);
1904
1905	/* configure CMD1 and VLD for ONFI param probing in QPIC v1 */
1906	if (!nandc->props->qpic_version2) {
1907	nandc->regs->vld = cpu_to_le32((nandc->vld & ~READ_START_VLD));
1908	nandc->regs->cmd1 = cpu_to_le32((nandc->cmd1 & ~READ_ADDR_MASK) |
1909	FIELD_PREP(READ_ADDR_MASK, NAND_CMD_PARAM));
1910	}
1911
1912	nandc->regs->exec = cpu_to_le32(1);
1913
1914	if (!nandc->props->qpic_version2) {
1915	nandc->regs->orig_cmd1 = cpu_to_le32(nandc->cmd1);
1916	nandc->regs->orig_vld = cpu_to_le32(nandc->vld);
1917	}
1918
1919	instr = q_op.data_instr;
1920	op_id = q_op.data_instr_idx;
1921	len = nand_subop_get_data_len(subop, op_id);
1922
1923	if (nandc->props->qpic_version2)
1924	nandc_set_read_loc_last(chip, reg_base, cw_offset: 0, read_size: len, is_last_read_loc: 1);
1925	else
1926	nandc_set_read_loc_first(chip, reg_base, cw_offset: 0, read_size: len, is_last_read_loc: 1);
1927
1928	if (!nandc->props->qpic_version2) {
1929	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->vld, NAND_DEV_CMD_VLD, num_regs: 1, flags: 0);
1930	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->cmd1, NAND_DEV_CMD1, num_regs: 1, NAND_BAM_NEXT_SGL);
1931	}
1932
1933	nandc->buf_count = 512;
1934	memset(nandc->data_buffer, 0xff, nandc->buf_count);
1935
1936	config_nand_single_cw_page_read(chip, use_ecc: false, cw: 0);
1937
1938	qcom_read_data_dma(nandc, FLASH_BUF_ACC, vaddr: nandc->data_buffer,
1939	size: nandc->buf_count, flags: 0);
1940
1941	/* restore CMD1 and VLD regs */
1942	if (!nandc->props->qpic_version2) {
1943	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->orig_cmd1, NAND_DEV_CMD1_RESTORE, num_regs: 1, flags: 0);
1944	qcom_write_reg_dma(nandc, vaddr: &nandc->regs->orig_vld, NAND_DEV_CMD_VLD_RESTORE, num_regs: 1,
1945	NAND_BAM_NEXT_SGL);
1946	}
1947
1948	ret = qcom_submit_descs(nandc);
1949	if (ret) {
1950	dev_err(nandc->dev, "failure in submitting param page descriptor\n");
1951	goto err_out;
1952	}
1953
1954	ret = qcom_wait_rdy_poll(chip, time_ms: q_op.rdy_timeout_ms);
1955	if (ret)
1956	goto err_out;
1957
1958	memcpy(instr->ctx.data.buf.in, nandc->data_buffer, len);
1959
1960	err_out:
1961	return ret;
1962	}
1963
1964	static const struct nand_op_parser qcom_op_parser = NAND_OP_PARSER(
1965	NAND_OP_PARSER_PATTERN(
1966	qcom_read_id_type_exec,
1967	NAND_OP_PARSER_PAT_CMD_ELEM(false),
1968	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYCLE),
1969	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
1970	NAND_OP_PARSER_PATTERN(
1971	qcom_read_status_exec,
1972	NAND_OP_PARSER_PAT_CMD_ELEM(false),
1973	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
1974	NAND_OP_PARSER_PATTERN(
1975	qcom_param_page_type_exec,
1976	NAND_OP_PARSER_PAT_CMD_ELEM(false),
1977	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYCLE),
1978	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1979	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 512)),
1980	NAND_OP_PARSER_PATTERN(
1981	qcom_misc_cmd_type_exec,
1982	NAND_OP_PARSER_PAT_CMD_ELEM(false),
1983	NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYCLE),
1984	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1985	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1986	);
1987
1988	static int qcom_check_op(struct nand_chip *chip,
1989	const struct nand_operation *op)
1990	{
1991	const struct nand_op_instr *instr;
1992	int op_id;
1993
1994	for (op_id = 0; op_id < op->ninstrs; op_id++) {
1995	instr = &op->instrs[op_id];
1996
1997	switch (instr->type) {
1998	case NAND_OP_CMD_INSTR:
1999	if (instr->ctx.cmd.opcode != NAND_CMD_RESET &&
2000	instr->ctx.cmd.opcode != NAND_CMD_READID &&
2001	instr->ctx.cmd.opcode != NAND_CMD_PARAM &&
2002	instr->ctx.cmd.opcode != NAND_CMD_ERASE1 &&
2003	instr->ctx.cmd.opcode != NAND_CMD_ERASE2 &&
2004	instr->ctx.cmd.opcode != NAND_CMD_STATUS &&
2005	instr->ctx.cmd.opcode != NAND_CMD_PAGEPROG &&
2006	instr->ctx.cmd.opcode != NAND_CMD_READ0 &&
2007	instr->ctx.cmd.opcode != NAND_CMD_READSTART)
2008	return -EOPNOTSUPP;
2009	break;
2010	default:
2011	break;
2012	}
2013	}
2014
2015	return 0;
2016	}
2017
2018	static int qcom_nand_exec_op(struct nand_chip *chip,
2019	const struct nand_operation *op, bool check_only)
2020	{
2021	if (check_only)
2022	return qcom_check_op(chip, op);
2023
2024	return nand_op_parser_exec_op(chip, parser: &qcom_op_parser, op, check_only);
2025	}
2026
2027	static const struct nand_controller_ops qcom_nandc_ops = {
2028	.attach_chip = qcom_nand_attach_chip,
2029	.exec_op = qcom_nand_exec_op,
2030	};
2031
2032	/* one time setup of a few nand controller registers */
2033	static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2034	{
2035	u32 nand_ctrl;
2036
2037	nand_controller_init(nfc: nandc->controller);
2038	nandc->controller->ops = &qcom_nandc_ops;
2039
2040	/* kill onenand */
2041	if (!nandc->props->nandc_part_of_qpic)
2042	nandc_write(nandc, SFLASHC_BURST_CFG, val: 0);
2043
2044	if (!nandc->props->qpic_version2)
2045	nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2046	NAND_DEV_CMD_VLD_VAL);
2047
2048	/* enable ADM or BAM DMA */
2049	if (nandc->props->supports_bam) {
2050	nand_ctrl = nandc_read(nandc, NAND_CTRL);
2051
2052	/*
2053	*NAND_CTRL is an operational registers, and CPU
2054	* access to operational registers are read only
2055	* in BAM mode. So update the NAND_CTRL register
2056	* only if it is not in BAM mode. In most cases BAM
2057	* mode will be enabled in bootloader
2058	*/
2059	if (!(nand_ctrl & BAM_MODE_EN))
2060	nandc_write(nandc, NAND_CTRL, val: nand_ctrl | BAM_MODE_EN);
2061	} else {
2062	nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2063	}
2064
2065	/* save the original values of these registers */
2066	if (!nandc->props->qpic_version2) {
2067	nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2068	nandc->vld = NAND_DEV_CMD_VLD_VAL;
2069	}
2070
2071	return 0;
2072	}
2073
2074	static const char * const probes[] = { "cmdlinepart", "ofpart", "qcomsmem", NULL };
2075
2076	static int qcom_nand_host_parse_boot_partitions(struct qcom_nand_controller *nandc,
2077	struct qcom_nand_host *host,
2078	struct device_node *dn)
2079	{
2080	struct nand_chip *chip = &host->chip;
2081	struct mtd_info *mtd = nand_to_mtd(chip);
2082	struct qcom_nand_boot_partition *boot_partition;
2083	struct device *dev = nandc->dev;
2084	int partitions_count, i, j, ret;
2085
2086	if (!of_property_present(np: dn, propname: "qcom,boot-partitions"))
2087	return 0;
2088
2089	partitions_count = of_property_count_u32_elems(np: dn, propname: "qcom,boot-partitions");
2090	if (partitions_count <= 0) {
2091	dev_err(dev, "Error parsing boot partition\n");
2092	return partitions_count ? partitions_count : -EINVAL;
2093	}
2094
2095	host->nr_boot_partitions = partitions_count / 2;
2096	host->boot_partitions = devm_kcalloc(dev, n: host->nr_boot_partitions,
2097	size: sizeof(*host->boot_partitions), GFP_KERNEL);
2098	if (!host->boot_partitions) {
2099	host->nr_boot_partitions = 0;
2100	return -ENOMEM;
2101	}
2102
2103	for (i = 0, j = 0; i < host->nr_boot_partitions; i++, j += 2) {
2104	boot_partition = &host->boot_partitions[i];
2105
2106	ret = of_property_read_u32_index(np: dn, propname: "qcom,boot-partitions", index: j,
2107	out_value: &boot_partition->page_offset);
2108	if (ret) {
2109	dev_err(dev, "Error parsing boot partition offset at index %d\n", i);
2110	host->nr_boot_partitions = 0;
2111	return ret;
2112	}
2113
2114	if (boot_partition->page_offset % mtd->writesize) {
2115	dev_err(dev, "Boot partition offset not multiple of writesize at index %i\n",
2116	i);
2117	host->nr_boot_partitions = 0;
2118	return -EINVAL;
2119	}
2120	/* Convert offset to nand pages */
2121	boot_partition->page_offset /= mtd->writesize;
2122
2123	ret = of_property_read_u32_index(np: dn, propname: "qcom,boot-partitions", index: j + 1,
2124	out_value: &boot_partition->page_size);
2125	if (ret) {
2126	dev_err(dev, "Error parsing boot partition size at index %d\n", i);
2127	host->nr_boot_partitions = 0;
2128	return ret;
2129	}
2130
2131	if (boot_partition->page_size % mtd->writesize) {
2132	dev_err(dev, "Boot partition size not multiple of writesize at index %i\n",
2133	i);
2134	host->nr_boot_partitions = 0;
2135	return -EINVAL;
2136	}
2137	/* Convert size to nand pages */
2138	boot_partition->page_size /= mtd->writesize;
2139	}
2140
2141	return 0;
2142	}
2143
2144	static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2145	struct qcom_nand_host *host,
2146	struct device_node *dn)
2147	{
2148	struct nand_chip *chip = &host->chip;
2149	struct mtd_info *mtd = nand_to_mtd(chip);
2150	struct device *dev = nandc->dev;
2151	int ret;
2152
2153	ret = of_property_read_u32(np: dn, propname: "reg", out_value: &host->cs);
2154	if (ret) {
2155	dev_err(dev, "can't get chip-select\n");
2156	return -ENXIO;
2157	}
2158
2159	nand_set_flash_node(chip, np: dn);
2160	mtd->name = devm_kasprintf(dev, GFP_KERNEL, fmt: "qcom_nand.%d", host->cs);
2161	if (!mtd->name)
2162	return -ENOMEM;
2163
2164	mtd->owner = THIS_MODULE;
2165	mtd->dev.parent = dev;
2166
2167	/*
2168	* the bad block marker is readable only when we read the last codeword
2169	* of a page with ECC disabled. currently, the nand_base and nand_bbt
2170	* helpers don't allow us to read BB from a nand chip with ECC
2171	* disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2172	* and block_markbad helpers until we permanently switch to using
2173	* MTD_OPS_RAW for all drivers (with the help of badblockbits)
2174	*/
2175	chip->legacy.block_bad = qcom_nandc_block_bad;
2176	chip->legacy.block_markbad = qcom_nandc_block_markbad;
2177
2178	chip->controller = nandc->controller;
2179	chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA |
2180	NAND_SKIP_BBTSCAN;
2181
2182	/* set up initial status value */
2183	host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2184
2185	ret = nand_scan(chip, max_chips: 1);
2186	if (ret)
2187	return ret;
2188
2189	ret = mtd_device_parse_register(mtd, part_probe_types: probes, NULL, NULL, defnr_parts: 0);
2190	if (ret)
2191	goto err;
2192
2193	if (nandc->props->use_codeword_fixup) {
2194	ret = qcom_nand_host_parse_boot_partitions(nandc, host, dn);
2195	if (ret)
2196	goto err;
2197	}
2198
2199	return 0;
2200
2201	err:
2202	nand_cleanup(chip);
2203	return ret;
2204	}
2205
2206	static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2207	{
2208	struct device *dev = nandc->dev;
2209	struct device_node *dn = dev->of_node, *child;
2210	struct qcom_nand_host *host;
2211	int ret = -ENODEV;
2212
2213	for_each_available_child_of_node(dn, child) {
2214	host = devm_kzalloc(dev, size: sizeof(*host), GFP_KERNEL);
2215	if (!host) {
2216	of_node_put(node: child);
2217	return -ENOMEM;
2218	}
2219
2220	ret = qcom_nand_host_init_and_register(nandc, host, dn: child);
2221	if (ret) {
2222	devm_kfree(dev, p: host);
2223	continue;
2224	}
2225
2226	list_add_tail(new: &host->node, head: &nandc->host_list);
2227	}
2228
2229	return ret;
2230	}
2231
2232	/* parse custom DT properties here */
2233	static int qcom_nandc_parse_dt(struct platform_device *pdev)
2234	{
2235	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2236	struct device_node *np = nandc->dev->of_node;
2237	int ret;
2238
2239	if (!nandc->props->supports_bam) {
2240	ret = of_property_read_u32(np, propname: "qcom,cmd-crci",
2241	out_value: &nandc->cmd_crci);
2242	if (ret) {
2243	dev_err(nandc->dev, "command CRCI unspecified\n");
2244	return ret;
2245	}
2246
2247	ret = of_property_read_u32(np, propname: "qcom,data-crci",
2248	out_value: &nandc->data_crci);
2249	if (ret) {
2250	dev_err(nandc->dev, "data CRCI unspecified\n");
2251	return ret;
2252	}
2253	}
2254
2255	return 0;
2256	}
2257
2258	static int qcom_nandc_probe(struct platform_device *pdev)
2259	{
2260	struct qcom_nand_controller *nandc;
2261	struct nand_controller *controller;
2262	const void *dev_data;
2263	struct device *dev = &pdev->dev;
2264	struct resource *res;
2265	int ret;
2266
2267	nandc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*nandc) + sizeof(*controller),
2268	GFP_KERNEL);
2269	if (!nandc)
2270	return -ENOMEM;
2271	controller = (struct nand_controller *)&nandc[1];
2272
2273	platform_set_drvdata(pdev, data: nandc);
2274	nandc->dev = dev;
2275	nandc->controller = controller;
2276
2277	dev_data = of_device_get_match_data(dev);
2278	if (!dev_data) {
2279	dev_err(&pdev->dev, "failed to get device data\n");
2280	return -ENODEV;
2281	}
2282
2283	nandc->props = dev_data;
2284
2285	nandc->core_clk = devm_clk_get(dev, id: "core");
2286	if (IS_ERR(ptr: nandc->core_clk))
2287	return PTR_ERR(ptr: nandc->core_clk);
2288
2289	nandc->aon_clk = devm_clk_get(dev, id: "aon");
2290	if (IS_ERR(ptr: nandc->aon_clk))
2291	return PTR_ERR(ptr: nandc->aon_clk);
2292
2293	ret = qcom_nandc_parse_dt(pdev);
2294	if (ret)
2295	return ret;
2296
2297	nandc->base = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
2298	if (IS_ERR(ptr: nandc->base))
2299	return PTR_ERR(ptr: nandc->base);
2300
2301	nandc->base_phys = res->start;
2302	nandc->base_dma = dma_map_resource(dev, phys_addr: res->start,
2303	size: resource_size(res),
2304	dir: DMA_BIDIRECTIONAL, attrs: 0);
2305	if (dma_mapping_error(dev, dma_addr: nandc->base_dma))
2306	return -ENXIO;
2307
2308	ret = clk_prepare_enable(clk: nandc->core_clk);
2309	if (ret)
2310	goto err_core_clk;
2311
2312	ret = clk_prepare_enable(clk: nandc->aon_clk);
2313	if (ret)
2314	goto err_aon_clk;
2315
2316	ret = qcom_nandc_alloc(nandc);
2317	if (ret)
2318	goto err_nandc_alloc;
2319
2320	ret = qcom_nandc_setup(nandc);
2321	if (ret)
2322	goto err_setup;
2323
2324	ret = qcom_probe_nand_devices(nandc);
2325	if (ret)
2326	goto err_setup;
2327
2328	return 0;
2329
2330	err_setup:
2331	qcom_nandc_unalloc(nandc);
2332	err_nandc_alloc:
2333	clk_disable_unprepare(clk: nandc->aon_clk);
2334	err_aon_clk:
2335	clk_disable_unprepare(clk: nandc->core_clk);
2336	err_core_clk:
2337	dma_unmap_resource(dev, addr: nandc->base_dma, size: resource_size(res),
2338	dir: DMA_BIDIRECTIONAL, attrs: 0);
2339	return ret;
2340	}
2341
2342	static void qcom_nandc_remove(struct platform_device *pdev)
2343	{
2344	struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
2345	struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2346	struct qcom_nand_host *host;
2347	struct nand_chip *chip;
2348	int ret;
2349
2350	list_for_each_entry(host, &nandc->host_list, node) {
2351	chip = &host->chip;
2352	ret = mtd_device_unregister(master: nand_to_mtd(chip));
2353	WARN_ON(ret);
2354	nand_cleanup(chip);
2355	}
2356
2357	qcom_nandc_unalloc(nandc);
2358
2359	clk_disable_unprepare(clk: nandc->aon_clk);
2360	clk_disable_unprepare(clk: nandc->core_clk);
2361
2362	dma_unmap_resource(dev: &pdev->dev, addr: nandc->base_dma, size: resource_size(res),
2363	dir: DMA_BIDIRECTIONAL, attrs: 0);
2364	}
2365
2366	static const struct qcom_nandc_props ipq806x_nandc_props = {
2367	.ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
2368	.supports_bam = false,
2369	.use_codeword_fixup = true,
2370	.dev_cmd_reg_start = 0x0,
2371	.bam_offset = 0x30000,
2372	};
2373
2374	static const struct qcom_nandc_props ipq4019_nandc_props = {
2375	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2376	.supports_bam = true,
2377	.nandc_part_of_qpic = true,
2378	.dev_cmd_reg_start = 0x0,
2379	.bam_offset = 0x30000,
2380	};
2381
2382	static const struct qcom_nandc_props ipq8074_nandc_props = {
2383	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2384	.supports_bam = true,
2385	.nandc_part_of_qpic = true,
2386	.dev_cmd_reg_start = 0x7000,
2387	.bam_offset = 0x30000,
2388	};
2389
2390	static const struct qcom_nandc_props sdx55_nandc_props = {
2391	.ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
2392	.supports_bam = true,
2393	.nandc_part_of_qpic = true,
2394	.qpic_version2 = true,
2395	.dev_cmd_reg_start = 0x7000,
2396	.bam_offset = 0x30000,
2397	};
2398
2399	/*
2400	* data will hold a struct pointer containing more differences once we support
2401	* more controller variants
2402	*/
2403	static const struct of_device_id qcom_nandc_of_match[] = {
2404	{
2405	.compatible = "qcom,ipq806x-nand",
2406	.data = &ipq806x_nandc_props,
2407	},
2408	{
2409	.compatible = "qcom,ipq4019-nand",
2410	.data = &ipq4019_nandc_props,
2411	},
2412	{
2413	.compatible = "qcom,ipq6018-nand",
2414	.data = &ipq8074_nandc_props,
2415	},
2416	{
2417	.compatible = "qcom,ipq8074-nand",
2418	.data = &ipq8074_nandc_props,
2419	},
2420	{
2421	.compatible = "qcom,sdx55-nand",
2422	.data = &sdx55_nandc_props,
2423	},
2424	{}
2425	};
2426	MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
2427
2428	static struct platform_driver qcom_nandc_driver = {
2429	.driver = {
2430	.name = "qcom-nandc",
2431	.of_match_table = qcom_nandc_of_match,
2432	},
2433	.probe = qcom_nandc_probe,
2434	.remove = qcom_nandc_remove,
2435	};
2436	module_platform_driver(qcom_nandc_driver);
2437
2438	MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
2439	MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
2440	MODULE_LICENSE("GPL v2");
2441