1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* USB4 specific functionality
4	*
5	* Copyright (C) 2019, Intel Corporation
6	* Authors: Mika Westerberg <mika.westerberg@linux.intel.com>
7	* Rajmohan Mani <rajmohan.mani@intel.com>
8	*/
9
10	#include <linux/delay.h>
11	#include <linux/ktime.h>
12	#include <linux/string_choices.h>
13	#include <linux/units.h>
14
15	#include "sb_regs.h"
16	#include "tb.h"
17
18	#define USB4_DATA_RETRIES 3
19	#define USB4_DATA_DWORDS 16
20
21	#define USB4_NVM_READ_OFFSET_MASK GENMASK(23, 2)
22	#define USB4_NVM_READ_OFFSET_SHIFT 2
23	#define USB4_NVM_READ_LENGTH_MASK GENMASK(27, 24)
24	#define USB4_NVM_READ_LENGTH_SHIFT 24
25
26	#define USB4_NVM_SET_OFFSET_MASK USB4_NVM_READ_OFFSET_MASK
27	#define USB4_NVM_SET_OFFSET_SHIFT USB4_NVM_READ_OFFSET_SHIFT
28
29	#define USB4_DROM_ADDRESS_MASK GENMASK(14, 2)
30	#define USB4_DROM_ADDRESS_SHIFT 2
31	#define USB4_DROM_SIZE_MASK GENMASK(19, 15)
32	#define USB4_DROM_SIZE_SHIFT 15
33
34	#define USB4_NVM_SECTOR_SIZE_MASK GENMASK(23, 0)
35
36	#define USB4_BA_LENGTH_MASK GENMASK(7, 0)
37	#define USB4_BA_INDEX_MASK GENMASK(15, 0)
38
39	enum usb4_ba_index {
40	USB4_BA_MAX_USB3 = 0x1,
41	USB4_BA_MIN_DP_AUX = 0x2,
42	USB4_BA_MIN_DP_MAIN = 0x3,
43	USB4_BA_MAX_PCIE = 0x4,
44	USB4_BA_MAX_HI = 0x5,
45	};
46
47	#define USB4_BA_VALUE_MASK GENMASK(31, 16)
48	#define USB4_BA_VALUE_SHIFT 16
49
50	/* Delays in us used with usb4_port_wait_for_bit() */
51	#define USB4_PORT_DELAY 50
52	#define USB4_PORT_SB_DELAY 1000
53
54	static int usb4_native_switch_op(struct tb_switch *sw, u16 opcode,
55	u32 *metadata, u8 *status,
56	const void *tx_data, size_t tx_dwords,
57	void *rx_data, size_t rx_dwords)
58	{
59	u32 val;
60	int ret;
61
62	if (metadata) {
63	ret = tb_sw_write(sw, buffer: metadata, space: TB_CFG_SWITCH, ROUTER_CS_25, length: 1);
64	if (ret)
65	return ret;
66	}
67	if (tx_dwords) {
68	ret = tb_sw_write(sw, buffer: tx_data, space: TB_CFG_SWITCH, ROUTER_CS_9,
69	length: tx_dwords);
70	if (ret)
71	return ret;
72	}
73
74	val = opcode | ROUTER_CS_26_OV;
75	ret = tb_sw_write(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_26, length: 1);
76	if (ret)
77	return ret;
78
79	ret = tb_switch_wait_for_bit(sw, ROUTER_CS_26, ROUTER_CS_26_OV, value: 0, timeout_msec: 500);
80	if (ret)
81	return ret;
82
83	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_26, length: 1);
84	if (ret)
85	return ret;
86
87	if (val & ROUTER_CS_26_ONS)
88	return -EOPNOTSUPP;
89
90	if (status)
91	*status = (val & ROUTER_CS_26_STATUS_MASK) >>
92	ROUTER_CS_26_STATUS_SHIFT;
93
94	if (metadata) {
95	ret = tb_sw_read(sw, buffer: metadata, space: TB_CFG_SWITCH, ROUTER_CS_25, length: 1);
96	if (ret)
97	return ret;
98	}
99	if (rx_dwords) {
100	ret = tb_sw_read(sw, buffer: rx_data, space: TB_CFG_SWITCH, ROUTER_CS_9,
101	length: rx_dwords);
102	if (ret)
103	return ret;
104	}
105
106	return 0;
107	}
108
109	static int __usb4_switch_op(struct tb_switch *sw, u16 opcode, u32 *metadata,
110	u8 *status, const void *tx_data, size_t tx_dwords,
111	void *rx_data, size_t rx_dwords)
112	{
113	const struct tb_cm_ops *cm_ops = sw->tb->cm_ops;
114
115	if (tx_dwords > USB4_DATA_DWORDS || rx_dwords > USB4_DATA_DWORDS)
116	return -EINVAL;
117
118	/*
119	* If the connection manager implementation provides USB4 router
120	* operation proxy callback, call it here instead of running the
121	* operation natively.
122	*/
123	if (cm_ops->usb4_switch_op) {
124	int ret;
125
126	ret = cm_ops->usb4_switch_op(sw, opcode, metadata, status,
127	tx_data, tx_dwords, rx_data,
128	rx_dwords);
129	if (ret != -EOPNOTSUPP)
130	return ret;
131
132	/*
133	* If the proxy was not supported then run the native
134	* router operation instead.
135	*/
136	}
137
138	return usb4_native_switch_op(sw, opcode, metadata, status, tx_data,
139	tx_dwords, rx_data, rx_dwords);
140	}
141
142	static inline int usb4_switch_op(struct tb_switch *sw, u16 opcode,
143	u32 *metadata, u8 *status)
144	{
145	return __usb4_switch_op(sw, opcode, metadata, status, NULL, tx_dwords: 0, NULL, rx_dwords: 0);
146	}
147
148	static inline int usb4_switch_op_data(struct tb_switch *sw, u16 opcode,
149	u32 *metadata, u8 *status,
150	const void *tx_data, size_t tx_dwords,
151	void *rx_data, size_t rx_dwords)
152	{
153	return __usb4_switch_op(sw, opcode, metadata, status, tx_data,
154	tx_dwords, rx_data, rx_dwords);
155	}
156
157	/**
158	* usb4_switch_check_wakes() - Check for wakes and notify PM core about them
159	* @sw: Router whose wakes to check
160	*
161	* Checks wakes occurred during suspend and notify the PM core about them.
162	*/
163	void usb4_switch_check_wakes(struct tb_switch *sw)
164	{
165	bool wakeup_usb4 = false;
166	struct usb4_port *usb4;
167	struct tb_port *port;
168	bool wakeup = false;
169	u32 val;
170
171	if (tb_route(sw)) {
172	if (tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_6, length: 1))
173	return;
174
175	tb_sw_dbg(sw, "PCIe wake: %s, USB3 wake: %s\n",
176	str_yes_no(val & ROUTER_CS_6_WOPS),
177	str_yes_no(val & ROUTER_CS_6_WOUS));
178
179	wakeup = val & (ROUTER_CS_6_WOPS | ROUTER_CS_6_WOUS);
180	}
181
182	/*
183	* Check for any downstream ports for USB4 wake,
184	* connection wake and disconnection wake.
185	*/
186	tb_switch_for_each_port(sw, port) {
187	if (!port->cap_usb4)
188	continue;
189
190	if (tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
191	offset: port->cap_usb4 + PORT_CS_18, length: 1))
192	break;
193
194	tb_port_dbg(port, "USB4 wake: %s, connection wake: %s, disconnection wake: %s\n",
195	str_yes_no(val & PORT_CS_18_WOU4S),
196	str_yes_no(val & PORT_CS_18_WOCS),
197	str_yes_no(val & PORT_CS_18_WODS));
198
199	wakeup_usb4 = val & (PORT_CS_18_WOU4S | PORT_CS_18_WOCS |
200	PORT_CS_18_WODS);
201
202	usb4 = port->usb4;
203	if (device_may_wakeup(dev: &usb4->dev) && wakeup_usb4)
204	pm_wakeup_event(dev: &usb4->dev, msec: 0);
205
206	wakeup |= wakeup_usb4;
207	}
208
209	if (wakeup)
210	pm_wakeup_event(dev: &sw->dev, msec: 0);
211	}
212
213	static bool link_is_usb4(struct tb_port *port)
214	{
215	u32 val;
216
217	if (!port->cap_usb4)
218	return false;
219
220	if (tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
221	offset: port->cap_usb4 + PORT_CS_18, length: 1))
222	return false;
223
224	return !(val & PORT_CS_18_TCM);
225	}
226
227	/**
228	* usb4_switch_setup() - Additional setup for USB4 device
229	* @sw: USB4 router to setup
230	*
231	* USB4 routers need additional settings in order to enable all the
232	* tunneling. This function enables USB and PCIe tunneling if it can be
233	* enabled (e.g the parent switch also supports them). If USB tunneling
234	* is not available for some reason (like that there is Thunderbolt 3
235	* switch upstream) then the internal xHCI controller is enabled
236	* instead.
237	*
238	* This does not set the configuration valid bit of the router. To do
239	* that call usb4_switch_configuration_valid().
240	*
241	* Return: %0 on success, negative errno otherwise.
242	*/
243	int usb4_switch_setup(struct tb_switch *sw)
244	{
245	struct tb_switch *parent = tb_switch_parent(sw);
246	struct tb_port *down;
247	bool tbt3, xhci;
248	u32 val = 0;
249	int ret;
250
251	if (!tb_route(sw))
252	return 0;
253
254	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_6, length: 1);
255	if (ret)
256	return ret;
257
258	down = tb_switch_downstream_port(sw);
259	sw->link_usb4 = link_is_usb4(port: down);
260	tb_sw_dbg(sw, "link: %s\n", sw->link_usb4 ? "USB4" : "TBT");
261
262	xhci = val & ROUTER_CS_6_HCI;
263	tbt3 = !(val & ROUTER_CS_6_TNS);
264
265	tb_sw_dbg(sw, "TBT3 support: %s, xHCI: %s\n",
266	str_yes_no(tbt3), str_yes_no(xhci));
267
268	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
269	if (ret)
270	return ret;
271
272	if (tb_acpi_may_tunnel_usb3() && sw->link_usb4 &&
273	tb_switch_find_port(sw: parent, type: TB_TYPE_USB3_DOWN)) {
274	val |= ROUTER_CS_5_UTO;
275	xhci = false;
276	}
277
278	/*
279	* Only enable PCIe tunneling if the parent router supports it
280	* and it is not disabled.
281	*/
282	if (tb_acpi_may_tunnel_pcie() &&
283	tb_switch_find_port(sw: parent, type: TB_TYPE_PCIE_DOWN)) {
284	val |= ROUTER_CS_5_PTO;
285	/*
286	* xHCI can be enabled if PCIe tunneling is supported
287	* and the parent does not have any USB3 downstream
288	* adapters (so we cannot do USB 3.x tunneling).
289	*/
290	if (xhci)
291	val |= ROUTER_CS_5_HCO;
292	}
293
294	/* TBT3 supported by the CM */
295	val &= ~ROUTER_CS_5_CNS;
296
297	return tb_sw_write(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
298	}
299
300	/**
301	* usb4_switch_configuration_valid() - Set tunneling configuration to be valid
302	* @sw: USB4 router
303	*
304	* Sets configuration valid bit for the router. Must be called before
305	* any tunnels can be set through the router and after
306	* usb4_switch_setup() has been called. Can be called to host and device
307	* routers (does nothing for the latter).
308	*
309	* Return: %0 on success, negative errno otherwise.
310	*/
311	int usb4_switch_configuration_valid(struct tb_switch *sw)
312	{
313	u32 val;
314	int ret;
315
316	if (!tb_route(sw))
317	return 0;
318
319	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
320	if (ret)
321	return ret;
322
323	val |= ROUTER_CS_5_CV;
324
325	ret = tb_sw_write(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
326	if (ret)
327	return ret;
328
329	return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_CR,
330	ROUTER_CS_6_CR, timeout_msec: 50);
331	}
332
333	/**
334	* usb4_switch_read_uid() - Read UID from USB4 router
335	* @sw: USB4 router
336	* @uid: UID is stored here
337	*
338	* Reads 64-bit UID from USB4 router config space.
339	*
340	* Return: %0 on success, negative errno otherwise.
341	*/
342	int usb4_switch_read_uid(struct tb_switch *sw, u64 *uid)
343	{
344	return tb_sw_read(sw, buffer: uid, space: TB_CFG_SWITCH, ROUTER_CS_7, length: 2);
345	}
346
347	static int usb4_switch_drom_read_block(void *data,
348	unsigned int dwaddress, void *buf,
349	size_t dwords)
350	{
351	struct tb_switch *sw = data;
352	u8 status = 0;
353	u32 metadata;
354	int ret;
355
356	metadata = (dwords << USB4_DROM_SIZE_SHIFT) & USB4_DROM_SIZE_MASK;
357	metadata |= (dwaddress << USB4_DROM_ADDRESS_SHIFT) &
358	USB4_DROM_ADDRESS_MASK;
359
360	ret = usb4_switch_op_data(sw, opcode: USB4_SWITCH_OP_DROM_READ, metadata: &metadata,
361	status: &status, NULL, tx_dwords: 0, rx_data: buf, rx_dwords: dwords);
362	if (ret)
363	return ret;
364
365	return status ? -EIO : 0;
366	}
367
368	/**
369	* usb4_switch_drom_read() - Read arbitrary bytes from USB4 router DROM
370	* @sw: USB4 router
371	* @address: Byte address inside DROM to start reading
372	* @buf: Buffer where the DROM content is stored
373	* @size: Number of bytes to read from DROM
374	*
375	* Uses USB4 router operations to read router DROM. For devices this
376	* should always work but for hosts it may return %-EOPNOTSUPP in which
377	* case the host router does not have DROM.
378	*
379	* Return: %0 on success, negative errno otherwise.
380	*/
381	int usb4_switch_drom_read(struct tb_switch *sw, unsigned int address, void *buf,
382	size_t size)
383	{
384	return tb_nvm_read_data(address, buf, size, USB4_DATA_RETRIES,
385	read_block: usb4_switch_drom_read_block, read_block_data: sw);
386	}
387
388	/**
389	* usb4_switch_lane_bonding_possible() - Are conditions met for lane bonding
390	* @sw: USB4 router
391	*
392	* Checks whether conditions are met so that lane bonding can be
393	* established with the upstream router. Call only for device routers.
394	*
395	* Return: %true if lane bonding is possible, %false otherwise.
396	*/
397	bool usb4_switch_lane_bonding_possible(struct tb_switch *sw)
398	{
399	struct tb_port *up;
400	int ret;
401	u32 val;
402
403	up = tb_upstream_port(sw);
404	ret = tb_port_read(port: up, buffer: &val, space: TB_CFG_PORT, offset: up->cap_usb4 + PORT_CS_18, length: 1);
405	if (ret)
406	return false;
407
408	return !!(val & PORT_CS_18_BE);
409	}
410
411	/**
412	* usb4_switch_set_wake() - Enabled/disable wake
413	* @sw: USB4 router
414	* @flags: Wakeup flags (%0 to disable)
415	* @runtime: Wake is being programmed during system runtime
416	*
417	* Enables/disables router to wake up from sleep.
418	*
419	* Return: %0 on success, negative errno otherwise.
420	*/
421	int usb4_switch_set_wake(struct tb_switch *sw, unsigned int flags, bool runtime)
422	{
423	struct tb_port *port;
424	u64 route = tb_route(sw);
425	u32 val;
426	int ret;
427
428	/*
429	* Enable wakes coming from all USB4 downstream ports (from
430	* child routers). For device routers do this also for the
431	* upstream USB4 port.
432	*/
433	tb_switch_for_each_port(sw, port) {
434	if (!tb_port_is_null(port))
435	continue;
436	if (!route && tb_is_upstream_port(port))
437	continue;
438	if (!port->cap_usb4)
439	continue;
440
441	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
442	offset: port->cap_usb4 + PORT_CS_19, length: 1);
443	if (ret)
444	return ret;
445
446	val &= ~(PORT_CS_19_WOC | PORT_CS_19_WOD | PORT_CS_19_WOU4);
447
448	if (tb_is_upstream_port(port)) {
449	val |= PORT_CS_19_WOU4;
450	} else {
451	bool configured = val & PORT_CS_19_PC;
452	bool wakeup = runtime || device_may_wakeup(dev: &port->usb4->dev);
453
454	if ((flags & TB_WAKE_ON_CONNECT) && wakeup && !configured)
455	val |= PORT_CS_19_WOC;
456	if ((flags & TB_WAKE_ON_DISCONNECT) && wakeup && configured)
457	val |= PORT_CS_19_WOD;
458	if ((flags & TB_WAKE_ON_USB4) && configured)
459	val |= PORT_CS_19_WOU4;
460	}
461
462	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
463	offset: port->cap_usb4 + PORT_CS_19, length: 1);
464	if (ret)
465	return ret;
466	}
467
468	/*
469	* Enable wakes from PCIe, USB 3.x and DP on this router. Only
470	* needed for device routers.
471	*/
472	if (route) {
473	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
474	if (ret)
475	return ret;
476
477	val &= ~(ROUTER_CS_5_WOP | ROUTER_CS_5_WOU | ROUTER_CS_5_WOD);
478	if (flags & TB_WAKE_ON_USB3)
479	val |= ROUTER_CS_5_WOU;
480	if (flags & TB_WAKE_ON_PCIE)
481	val |= ROUTER_CS_5_WOP;
482	if (flags & TB_WAKE_ON_DP)
483	val |= ROUTER_CS_5_WOD;
484
485	ret = tb_sw_write(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
486	if (ret)
487	return ret;
488	}
489
490	return 0;
491	}
492
493	/**
494	* usb4_switch_set_sleep() - Prepare the router to enter sleep
495	* @sw: USB4 router
496	*
497	* Sets sleep bit for the router and waits until router sleep ready
498	* bit has been asserted.
499	*
500	* Return: %0 on success, negative errno otherwise.
501	*/
502	int usb4_switch_set_sleep(struct tb_switch *sw)
503	{
504	int ret;
505	u32 val;
506
507	/* Set sleep bit and wait for sleep ready to be asserted */
508	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
509	if (ret)
510	return ret;
511
512	val |= ROUTER_CS_5_SLP;
513
514	ret = tb_sw_write(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_5, length: 1);
515	if (ret)
516	return ret;
517
518	return tb_switch_wait_for_bit(sw, ROUTER_CS_6, ROUTER_CS_6_SLPR,
519	ROUTER_CS_6_SLPR, timeout_msec: 500);
520	}
521
522	/**
523	* usb4_switch_nvm_sector_size() - Return router NVM sector size
524	* @sw: USB4 router
525	*
526	* Return:
527	* * NVM sector size in bytes if router supports NVM operations.
528	* * %-EOPNOTSUPP - If router does not support NVM operations.
529	* * Negative errno - Another error occurred.
530	*/
531	int usb4_switch_nvm_sector_size(struct tb_switch *sw)
532	{
533	u32 metadata;
534	u8 status;
535	int ret;
536
537	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_NVM_SECTOR_SIZE, metadata: &metadata,
538	status: &status);
539	if (ret)
540	return ret;
541
542	if (status)
543	return status == 0x2 ? -EOPNOTSUPP : -EIO;
544
545	return metadata & USB4_NVM_SECTOR_SIZE_MASK;
546	}
547
548	static int usb4_switch_nvm_read_block(void *data,
549	unsigned int dwaddress, void *buf, size_t dwords)
550	{
551	struct tb_switch *sw = data;
552	u8 status = 0;
553	u32 metadata;
554	int ret;
555
556	metadata = (dwords << USB4_NVM_READ_LENGTH_SHIFT) &
557	USB4_NVM_READ_LENGTH_MASK;
558	metadata |= (dwaddress << USB4_NVM_READ_OFFSET_SHIFT) &
559	USB4_NVM_READ_OFFSET_MASK;
560
561	ret = usb4_switch_op_data(sw, opcode: USB4_SWITCH_OP_NVM_READ, metadata: &metadata,
562	status: &status, NULL, tx_dwords: 0, rx_data: buf, rx_dwords: dwords);
563	if (ret)
564	return ret;
565
566	return status ? -EIO : 0;
567	}
568
569	/**
570	* usb4_switch_nvm_read() - Read arbitrary bytes from router NVM
571	* @sw: USB4 router
572	* @address: Starting address in bytes
573	* @buf: Read data is placed here
574	* @size: How many bytes to read
575	*
576	* Reads NVM contents of the router.
577	*
578	* Return:
579	* * %0 - Read completed successfully.
580	* * %-EOPNOTSUPP - NVM not supported.
581	* * Negative errno - Another error occurred.
582	*/
583	int usb4_switch_nvm_read(struct tb_switch *sw, unsigned int address, void *buf,
584	size_t size)
585	{
586	return tb_nvm_read_data(address, buf, size, USB4_DATA_RETRIES,
587	read_block: usb4_switch_nvm_read_block, read_block_data: sw);
588	}
589
590	/**
591	* usb4_switch_nvm_set_offset() - Set NVM write offset
592	* @sw: USB4 router
593	* @address: Start offset
594	*
595	* Explicitly sets NVM write offset. Normally when writing to NVM this
596	* is done automatically by usb4_switch_nvm_write().
597	*
598	* Return: %0 on success, negative errno otherwise.
599	*/
600	int usb4_switch_nvm_set_offset(struct tb_switch *sw, unsigned int address)
601	{
602	u32 metadata, dwaddress;
603	u8 status = 0;
604	int ret;
605
606	dwaddress = address / 4;
607	metadata = (dwaddress << USB4_NVM_SET_OFFSET_SHIFT) &
608	USB4_NVM_SET_OFFSET_MASK;
609
610	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_NVM_SET_OFFSET, metadata: &metadata,
611	status: &status);
612	if (ret)
613	return ret;
614
615	return status ? -EIO : 0;
616	}
617
618	static int usb4_switch_nvm_write_next_block(void *data, unsigned int dwaddress,
619	const void *buf, size_t dwords)
620	{
621	struct tb_switch *sw = data;
622	u8 status;
623	int ret;
624
625	ret = usb4_switch_op_data(sw, opcode: USB4_SWITCH_OP_NVM_WRITE, NULL, status: &status,
626	tx_data: buf, tx_dwords: dwords, NULL, rx_dwords: 0);
627	if (ret)
628	return ret;
629
630	return status ? -EIO : 0;
631	}
632
633	/**
634	* usb4_switch_nvm_write() - Write to the router NVM
635	* @sw: USB4 router
636	* @address: Start address where to write in bytes
637	* @buf: Pointer to the data to write
638	* @size: Size of @buf in bytes
639	*
640	* Writes @buf to the router NVM using USB4 router operations.
641	*
642	* Return:
643	* * %0 - Write completed successfully.
644	* * %-EOPNOTSUPP - NVM write not supported.
645	* * Negative errno - Another error occurred.
646	*/
647	int usb4_switch_nvm_write(struct tb_switch *sw, unsigned int address,
648	const void *buf, size_t size)
649	{
650	int ret;
651
652	ret = usb4_switch_nvm_set_offset(sw, address);
653	if (ret)
654	return ret;
655
656	return tb_nvm_write_data(address, buf, size, USB4_DATA_RETRIES,
657	write_next_block: usb4_switch_nvm_write_next_block, write_block_data: sw);
658	}
659
660	/**
661	* usb4_switch_nvm_authenticate() - Authenticate new NVM
662	* @sw: USB4 router
663	*
664	* After the new NVM has been written via usb4_switch_nvm_write(), this
665	* function triggers NVM authentication process. The router gets power
666	* cycled and if the authentication is successful the new NVM starts
667	* running.
668	*
669	* The caller should call usb4_switch_nvm_authenticate_status() to read
670	* the status of the authentication after power cycle. It should be the
671	* first router operation to avoid the status being lost.
672	*
673	* Return: %0 on success, negative errno otherwise.
674	*/
675	int usb4_switch_nvm_authenticate(struct tb_switch *sw)
676	{
677	int ret;
678
679	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_NVM_AUTH, NULL, NULL);
680	switch (ret) {
681	/*
682	* The router is power cycled once NVM_AUTH is started so it is
683	* expected to get any of the following errors back.
684	*/
685	case -EACCES:
686	case -ENOTCONN:
687	case -ETIMEDOUT:
688	return 0;
689
690	default:
691	return ret;
692	}
693	}
694
695	/**
696	* usb4_switch_nvm_authenticate_status() - Read status of last NVM authenticate
697	* @sw: USB4 router
698	* @status: Status code of the operation
699	*
700	* The function checks if there is status available from the last NVM
701	* authenticate router operation.
702	*
703	* Must be called before any other router operation.
704	*
705	* Return:
706	* * %0 - If there is status. Status code is placed in @status.
707	* * Negative errno - Failure occurred.
708	*/
709	int usb4_switch_nvm_authenticate_status(struct tb_switch *sw, u32 *status)
710	{
711	const struct tb_cm_ops *cm_ops = sw->tb->cm_ops;
712	u16 opcode;
713	u32 val;
714	int ret;
715
716	if (cm_ops->usb4_switch_nvm_authenticate_status) {
717	ret = cm_ops->usb4_switch_nvm_authenticate_status(sw, status);
718	if (ret != -EOPNOTSUPP)
719	return ret;
720	}
721
722	ret = tb_sw_read(sw, buffer: &val, space: TB_CFG_SWITCH, ROUTER_CS_26, length: 1);
723	if (ret)
724	return ret;
725
726	/* Check that the opcode is correct */
727	opcode = val & ROUTER_CS_26_OPCODE_MASK;
728	if (opcode == USB4_SWITCH_OP_NVM_AUTH) {
729	if (val & ROUTER_CS_26_OV)
730	return -EBUSY;
731	if (val & ROUTER_CS_26_ONS)
732	return -EOPNOTSUPP;
733
734	*status = (val & ROUTER_CS_26_STATUS_MASK) >>
735	ROUTER_CS_26_STATUS_SHIFT;
736	} else {
737	*status = 0;
738	}
739
740	return 0;
741	}
742
743	/**
744	* usb4_switch_credits_init() - Read buffer allocation parameters
745	* @sw: USB4 router
746	*
747	* Reads @sw buffer allocation parameters and initializes @sw buffer
748	* allocation fields accordingly. Specifically @sw->credits_allocation
749	* is set to %true if these parameters can be used in tunneling.
750	*
751	* Return: %0 on success, negative errno otherwise.
752	*/
753	int usb4_switch_credits_init(struct tb_switch *sw)
754	{
755	int max_usb3, min_dp_aux, min_dp_main, max_pcie, max_dma;
756	int ret, length, i, nports;
757	const struct tb_port *port;
758	u32 data[USB4_DATA_DWORDS];
759	u32 metadata = 0;
760	u8 status = 0;
761
762	memset(data, 0, sizeof(data));
763	ret = usb4_switch_op_data(sw, opcode: USB4_SWITCH_OP_BUFFER_ALLOC, metadata: &metadata,
764	status: &status, NULL, tx_dwords: 0, rx_data: data, ARRAY_SIZE(data));
765	if (ret)
766	return ret;
767	if (status)
768	return -EIO;
769
770	length = metadata & USB4_BA_LENGTH_MASK;
771	if (WARN_ON(length > ARRAY_SIZE(data)))
772	return -EMSGSIZE;
773
774	max_usb3 = -1;
775	min_dp_aux = -1;
776	min_dp_main = -1;
777	max_pcie = -1;
778	max_dma = -1;
779
780	tb_sw_dbg(sw, "credit allocation parameters:\n");
781
782	for (i = 0; i < length; i++) {
783	u16 index, value;
784
785	index = data[i] & USB4_BA_INDEX_MASK;
786	value = (data[i] & USB4_BA_VALUE_MASK) >> USB4_BA_VALUE_SHIFT;
787
788	switch (index) {
789	case USB4_BA_MAX_USB3:
790	tb_sw_dbg(sw, " USB3: %u\n", value);
791	max_usb3 = value;
792	break;
793	case USB4_BA_MIN_DP_AUX:
794	tb_sw_dbg(sw, " DP AUX: %u\n", value);
795	min_dp_aux = value;
796	break;
797	case USB4_BA_MIN_DP_MAIN:
798	tb_sw_dbg(sw, " DP main: %u\n", value);
799	min_dp_main = value;
800	break;
801	case USB4_BA_MAX_PCIE:
802	tb_sw_dbg(sw, " PCIe: %u\n", value);
803	max_pcie = value;
804	break;
805	case USB4_BA_MAX_HI:
806	tb_sw_dbg(sw, " DMA: %u\n", value);
807	max_dma = value;
808	break;
809	default:
810	tb_sw_dbg(sw, " unknown credit allocation index %#x, skipping\n",
811	index);
812	break;
813	}
814	}
815
816	/*
817	* Validate the buffer allocation preferences. If we find
818	* issues, log a warning and fall back using the hard-coded
819	* values.
820	*/
821
822	/* Host router must report baMaxHI */
823	if (!tb_route(sw) && max_dma < 0) {
824	tb_sw_warn(sw, "host router is missing baMaxHI\n");
825	goto err_invalid;
826	}
827
828	nports = 0;
829	tb_switch_for_each_port(sw, port) {
830	if (tb_port_is_null(port))
831	nports++;
832	}
833
834	/* Must have DP buffer allocation (multiple USB4 ports) */
835	if (nports > 2 && (min_dp_aux < 0 || min_dp_main < 0)) {
836	tb_sw_warn(sw, "multiple USB4 ports require baMinDPaux/baMinDPmain\n");
837	goto err_invalid;
838	}
839
840	tb_switch_for_each_port(sw, port) {
841	if (tb_port_is_dpout(port) && min_dp_main < 0) {
842	tb_sw_warn(sw, "missing baMinDPmain");
843	goto err_invalid;
844	}
845	if ((tb_port_is_dpin(port) || tb_port_is_dpout(port)) &&
846	min_dp_aux < 0) {
847	tb_sw_warn(sw, "missing baMinDPaux");
848	goto err_invalid;
849	}
850	if ((tb_port_is_usb3_down(port) || tb_port_is_usb3_up(port)) &&
851	max_usb3 < 0) {
852	tb_sw_warn(sw, "missing baMaxUSB3");
853	goto err_invalid;
854	}
855	if ((tb_port_is_pcie_down(port) || tb_port_is_pcie_up(port)) &&
856	max_pcie < 0) {
857	tb_sw_warn(sw, "missing baMaxPCIe");
858	goto err_invalid;
859	}
860	}
861
862	/*
863	* Buffer allocation passed the validation so we can use it in
864	* path creation.
865	*/
866	sw->credit_allocation = true;
867	if (max_usb3 > 0)
868	sw->max_usb3_credits = max_usb3;
869	if (min_dp_aux > 0)
870	sw->min_dp_aux_credits = min_dp_aux;
871	if (min_dp_main > 0)
872	sw->min_dp_main_credits = min_dp_main;
873	if (max_pcie > 0)
874	sw->max_pcie_credits = max_pcie;
875	if (max_dma > 0)
876	sw->max_dma_credits = max_dma;
877
878	return 0;
879
880	err_invalid:
881	return -EINVAL;
882	}
883
884	/**
885	* usb4_switch_query_dp_resource() - Query availability of DP IN resource
886	* @sw: USB4 router
887	* @in: DP IN adapter
888	*
889	* For DP tunneling this function can be used to query availability of
890	* DP IN resource.
891	*
892	* Return: %true if the resource is available for DP tunneling, %false
893	* otherwise.
894	*/
895	bool usb4_switch_query_dp_resource(struct tb_switch *sw, struct tb_port *in)
896	{
897	u32 metadata = in->port;
898	u8 status;
899	int ret;
900
901	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_QUERY_DP_RESOURCE, metadata: &metadata,
902	status: &status);
903	/*
904	* If DP resource allocation is not supported assume it is
905	* always available.
906	*/
907	if (ret == -EOPNOTSUPP)
908	return true;
909	if (ret)
910	return false;
911
912	return !status;
913	}
914
915	/**
916	* usb4_switch_alloc_dp_resource() - Allocate DP IN resource
917	* @sw: USB4 router
918	* @in: DP IN adapter
919	*
920	* Allocates DP IN resource for DP tunneling using USB4 router
921	* operations.
922	*
923	* Return:
924	* * %0 - Resource allocated successfully.
925	* * %-EBUSY - Resource is already allocated.
926	* * Negative errno - Other failure occurred.
927	*/
928	int usb4_switch_alloc_dp_resource(struct tb_switch *sw, struct tb_port *in)
929	{
930	u32 metadata = in->port;
931	u8 status;
932	int ret;
933
934	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_ALLOC_DP_RESOURCE, metadata: &metadata,
935	status: &status);
936	if (ret == -EOPNOTSUPP)
937	return 0;
938	if (ret)
939	return ret;
940
941	return status ? -EBUSY : 0;
942	}
943
944	/**
945	* usb4_switch_dealloc_dp_resource() - Releases allocated DP IN resource
946	* @sw: USB4 router
947	* @in: DP IN adapter
948	*
949	* Releases the previously allocated DP IN resource.
950	*
951	* Return: %0 on success, negative errno otherwise.
952	*/
953	int usb4_switch_dealloc_dp_resource(struct tb_switch *sw, struct tb_port *in)
954	{
955	u32 metadata = in->port;
956	u8 status;
957	int ret;
958
959	ret = usb4_switch_op(sw, opcode: USB4_SWITCH_OP_DEALLOC_DP_RESOURCE, metadata: &metadata,
960	status: &status);
961	if (ret == -EOPNOTSUPP)
962	return 0;
963	if (ret)
964	return ret;
965
966	return status ? -EIO : 0;
967	}
968
969	/**
970	* usb4_port_index() - Finds matching USB4 port index
971	* @sw: USB4 router
972	* @port: USB4 protocol or lane adapter
973	*
974	* Finds matching USB4 port index (starting from %0) that given @port goes
975	* through.
976	*/
977	int usb4_port_index(const struct tb_switch *sw, const struct tb_port *port)
978	{
979	struct tb_port *p;
980	int usb4_idx = 0;
981
982	/* Assume port is primary */
983	tb_switch_for_each_port(sw, p) {
984	if (!tb_port_is_null(port: p))
985	continue;
986	if (tb_is_upstream_port(port: p))
987	continue;
988	if (!p->link_nr) {
989	if (p == port)
990	break;
991	usb4_idx++;
992	}
993	}
994
995	return usb4_idx;
996	}
997
998	/**
999	* usb4_switch_map_pcie_down() - Map USB4 port to a PCIe downstream adapter
1000	* @sw: USB4 router
1001	* @port: USB4 port
1002	*
1003	* USB4 routers have direct mapping between USB4 ports and PCIe
1004	* downstream adapters where the PCIe topology is extended. This
1005	* function returns the corresponding downstream PCIe adapter or %NULL
1006	* if no such mapping was possible.
1007	*
1008	* Return: Pointer to &struct tb_port or %NULL if not found.
1009	*/
1010	struct tb_port *usb4_switch_map_pcie_down(struct tb_switch *sw,
1011	const struct tb_port *port)
1012	{
1013	int usb4_idx = usb4_port_index(sw, port);
1014	struct tb_port *p;
1015	int pcie_idx = 0;
1016
1017	/* Find PCIe down port matching usb4_port */
1018	tb_switch_for_each_port(sw, p) {
1019	if (!tb_port_is_pcie_down(port: p))
1020	continue;
1021
1022	if (pcie_idx == usb4_idx)
1023	return p;
1024
1025	pcie_idx++;
1026	}
1027
1028	return NULL;
1029	}
1030
1031	/**
1032	* usb4_switch_map_usb3_down() - Map USB4 port to a USB3 downstream adapter
1033	* @sw: USB4 router
1034	* @port: USB4 port
1035	*
1036	* USB4 routers have direct mapping between USB4 ports and USB 3.x
1037	* downstream adapters where the USB 3.x topology is extended. This
1038	* function returns the corresponding downstream USB 3.x adapter or
1039	* %NULL if no such mapping was possible.
1040	*
1041	* Return: Pointer to &struct tb_port or %NULL if not found.
1042	*/
1043	struct tb_port *usb4_switch_map_usb3_down(struct tb_switch *sw,
1044	const struct tb_port *port)
1045	{
1046	int usb4_idx = usb4_port_index(sw, port);
1047	struct tb_port *p;
1048	int usb_idx = 0;
1049
1050	/* Find USB3 down port matching usb4_port */
1051	tb_switch_for_each_port(sw, p) {
1052	if (!tb_port_is_usb3_down(port: p))
1053	continue;
1054
1055	if (usb_idx == usb4_idx)
1056	return p;
1057
1058	usb_idx++;
1059	}
1060
1061	return NULL;
1062	}
1063
1064	/**
1065	* usb4_switch_add_ports() - Add USB4 ports for this router
1066	* @sw: USB4 router
1067	*
1068	* For USB4 router finds all USB4 ports and registers devices for each.
1069	* Can be called to any router.
1070	*
1071	* Return: %0 on success, negative errno otherwise.
1072	*/
1073	int usb4_switch_add_ports(struct tb_switch *sw)
1074	{
1075	struct tb_port *port;
1076
1077	if (tb_switch_is_icm(sw) || !tb_switch_is_usb4(sw))
1078	return 0;
1079
1080	tb_switch_for_each_port(sw, port) {
1081	struct usb4_port *usb4;
1082
1083	if (!tb_port_is_null(port))
1084	continue;
1085	if (!port->cap_usb4)
1086	continue;
1087
1088	usb4 = usb4_port_device_add(port);
1089	if (IS_ERR(ptr: usb4)) {
1090	usb4_switch_remove_ports(sw);
1091	return PTR_ERR(ptr: usb4);
1092	}
1093
1094	port->usb4 = usb4;
1095	}
1096
1097	return 0;
1098	}
1099
1100	/**
1101	* usb4_switch_remove_ports() - Removes USB4 ports from this router
1102	* @sw: USB4 router
1103	*
1104	* Unregisters previously registered USB4 ports.
1105	*/
1106	void usb4_switch_remove_ports(struct tb_switch *sw)
1107	{
1108	struct tb_port *port;
1109
1110	tb_switch_for_each_port(sw, port) {
1111	if (port->usb4) {
1112	usb4_port_device_remove(usb4: port->usb4);
1113	port->usb4 = NULL;
1114	}
1115	}
1116	}
1117
1118	/**
1119	* usb4_port_unlock() - Unlock USB4 downstream port
1120	* @port: USB4 port to unlock
1121	*
1122	* Unlocks USB4 downstream port so that the connection manager can
1123	* access the router below this port.
1124	*
1125	* Return: %0 on success, negative errno otherwise.
1126	*/
1127	int usb4_port_unlock(struct tb_port *port)
1128	{
1129	int ret;
1130	u32 val;
1131
1132	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT, ADP_CS_4, length: 1);
1133	if (ret)
1134	return ret;
1135
1136	val &= ~ADP_CS_4_LCK;
1137	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT, ADP_CS_4, length: 1);
1138	}
1139
1140	/**
1141	* usb4_port_hotplug_enable() - Enables hotplug for a port
1142	* @port: USB4 port to operate on
1143	*
1144	* Enables hot plug events on a given port. This is only intended
1145	* to be used on lane, DP-IN, and DP-OUT adapters.
1146	*
1147	* Return: %0 on success, negative errno otherwise.
1148	*/
1149	int usb4_port_hotplug_enable(struct tb_port *port)
1150	{
1151	int ret;
1152	u32 val;
1153
1154	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT, ADP_CS_5, length: 1);
1155	if (ret)
1156	return ret;
1157
1158	val &= ~ADP_CS_5_DHP;
1159	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT, ADP_CS_5, length: 1);
1160	}
1161
1162	/**
1163	* usb4_port_reset() - Issue downstream port reset
1164	* @port: USB4 port to reset
1165	*
1166	* Issues downstream port reset to @port.
1167	*
1168	* Return: %0 on success, negative errno otherwise.
1169	*/
1170	int usb4_port_reset(struct tb_port *port)
1171	{
1172	int ret;
1173	u32 val;
1174
1175	if (!port->cap_usb4)
1176	return -EINVAL;
1177
1178	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1179	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1180	if (ret)
1181	return ret;
1182
1183	val |= PORT_CS_19_DPR;
1184
1185	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1186	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1187	if (ret)
1188	return ret;
1189
1190	fsleep(usecs: 10000);
1191
1192	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1193	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1194	if (ret)
1195	return ret;
1196
1197	val &= ~PORT_CS_19_DPR;
1198
1199	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1200	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1201	}
1202
1203	static int usb4_port_set_configured(struct tb_port *port, bool configured)
1204	{
1205	int ret;
1206	u32 val;
1207
1208	if (!port->cap_usb4)
1209	return -EINVAL;
1210
1211	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1212	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1213	if (ret)
1214	return ret;
1215
1216	if (configured)
1217	val |= PORT_CS_19_PC;
1218	else
1219	val &= ~PORT_CS_19_PC;
1220
1221	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1222	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1223	}
1224
1225	/**
1226	* usb4_port_configure() - Set USB4 port configured
1227	* @port: USB4 router
1228	*
1229	* Sets the USB4 link to be configured for power management purposes.
1230	*
1231	* Return: %0 on success, negative errno otherwise.
1232	*/
1233	int usb4_port_configure(struct tb_port *port)
1234	{
1235	return usb4_port_set_configured(port, configured: true);
1236	}
1237
1238	/**
1239	* usb4_port_unconfigure() - Set USB4 port unconfigured
1240	* @port: USB4 router
1241	*
1242	* Sets the USB4 link to be unconfigured for power management purposes.
1243	*
1244	* Return: %0 on success, negative errno otherwise.
1245	*/
1246	void usb4_port_unconfigure(struct tb_port *port)
1247	{
1248	usb4_port_set_configured(port, configured: false);
1249	}
1250
1251	static int usb4_set_xdomain_configured(struct tb_port *port, bool configured)
1252	{
1253	int ret;
1254	u32 val;
1255
1256	if (!port->cap_usb4)
1257	return -EINVAL;
1258
1259	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1260	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1261	if (ret)
1262	return ret;
1263
1264	if (configured)
1265	val |= PORT_CS_19_PID;
1266	else
1267	val &= ~PORT_CS_19_PID;
1268
1269	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1270	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1271	}
1272
1273	/**
1274	* usb4_port_configure_xdomain() - Configure port for XDomain
1275	* @port: USB4 port connected to another host
1276	* @xd: XDomain that is connected to the port
1277	*
1278	* Marks the USB4 port as being connected to another host and updates
1279	* the link type.
1280	*
1281	* Return: %0 on success, negative errno otherwise.
1282	*/
1283	int usb4_port_configure_xdomain(struct tb_port *port, struct tb_xdomain *xd)
1284	{
1285	xd->link_usb4 = link_is_usb4(port);
1286	return usb4_set_xdomain_configured(port, configured: true);
1287	}
1288
1289	/**
1290	* usb4_port_unconfigure_xdomain() - Unconfigure port for XDomain
1291	* @port: USB4 port that was connected to another host
1292	*
1293	* Clears USB4 port from being marked as XDomain.
1294	*/
1295	void usb4_port_unconfigure_xdomain(struct tb_port *port)
1296	{
1297	usb4_set_xdomain_configured(port, configured: false);
1298	}
1299
1300	static int usb4_port_wait_for_bit(struct tb_port *port, u32 offset, u32 bit,
1301	u32 value, int timeout_msec, unsigned long delay_usec)
1302	{
1303	ktime_t timeout = ktime_add_ms(kt: ktime_get(), msec: timeout_msec);
1304
1305	do {
1306	u32 val;
1307	int ret;
1308
1309	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT, offset, length: 1);
1310	if (ret)
1311	return ret;
1312
1313	if ((val & bit) == value)
1314	return 0;
1315
1316	fsleep(usecs: delay_usec);
1317	} while (ktime_before(cmp1: ktime_get(), cmp2: timeout));
1318
1319	return -ETIMEDOUT;
1320	}
1321
1322	static int usb4_port_read_data(struct tb_port *port, void *data, size_t dwords)
1323	{
1324	if (dwords > USB4_DATA_DWORDS)
1325	return -EINVAL;
1326
1327	return tb_port_read(port, buffer: data, space: TB_CFG_PORT, offset: port->cap_usb4 + PORT_CS_2,
1328	length: dwords);
1329	}
1330
1331	static int usb4_port_write_data(struct tb_port *port, const void *data,
1332	size_t dwords)
1333	{
1334	if (dwords > USB4_DATA_DWORDS)
1335	return -EINVAL;
1336
1337	return tb_port_write(port, buffer: data, space: TB_CFG_PORT, offset: port->cap_usb4 + PORT_CS_2,
1338	length: dwords);
1339	}
1340
1341	/**
1342	* usb4_port_sb_read() - Read from sideband register
1343	* @port: USB4 port to read
1344	* @target: Sideband target
1345	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1346	* @reg: Sideband register index
1347	* @buf: Buffer where the sideband data is copied
1348	* @size: Size of @buf
1349	*
1350	* Reads data from sideband register @reg and copies it into @buf.
1351	*
1352	* Return: %0 on success, negative errno otherwise.
1353	*/
1354	int usb4_port_sb_read(struct tb_port *port, enum usb4_sb_target target, u8 index,
1355	u8 reg, void *buf, u8 size)
1356	{
1357	size_t dwords = DIV_ROUND_UP(size, 4);
1358	int ret;
1359	u32 val;
1360
1361	if (!port->cap_usb4)
1362	return -EINVAL;
1363
1364	val = reg;
1365	val |= size << PORT_CS_1_LENGTH_SHIFT;
1366	val |= (target << PORT_CS_1_TARGET_SHIFT) & PORT_CS_1_TARGET_MASK;
1367	if (target == USB4_SB_TARGET_RETIMER)
1368	val |= (index << PORT_CS_1_RETIMER_INDEX_SHIFT);
1369	val |= PORT_CS_1_PND;
1370
1371	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1372	offset: port->cap_usb4 + PORT_CS_1, length: 1);
1373	if (ret)
1374	return ret;
1375
1376	ret = usb4_port_wait_for_bit(port, offset: port->cap_usb4 + PORT_CS_1,
1377	PORT_CS_1_PND, value: 0, timeout_msec: 500, USB4_PORT_SB_DELAY);
1378	if (ret)
1379	return ret;
1380
1381	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1382	offset: port->cap_usb4 + PORT_CS_1, length: 1);
1383	if (ret)
1384	return ret;
1385
1386	if (val & PORT_CS_1_NR)
1387	return -ENODEV;
1388	if (val & PORT_CS_1_RC)
1389	return -EIO;
1390
1391	return buf ? usb4_port_read_data(port, data: buf, dwords) : 0;
1392	}
1393
1394	/**
1395	* usb4_port_sb_write() - Write to sideband register
1396	* @port: USB4 port to write
1397	* @target: Sideband target
1398	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1399	* @reg: Sideband register index
1400	* @buf: Data to write
1401	* @size: Size of @buf
1402	*
1403	* Writes @buf to sideband register @reg.
1404	*
1405	* Return: %0 on success, negative errno otherwise.
1406	*/
1407	int usb4_port_sb_write(struct tb_port *port, enum usb4_sb_target target,
1408	u8 index, u8 reg, const void *buf, u8 size)
1409	{
1410	size_t dwords = DIV_ROUND_UP(size, 4);
1411	int ret;
1412	u32 val;
1413
1414	if (!port->cap_usb4)
1415	return -EINVAL;
1416
1417	if (buf) {
1418	ret = usb4_port_write_data(port, data: buf, dwords);
1419	if (ret)
1420	return ret;
1421	}
1422
1423	val = reg;
1424	val |= size << PORT_CS_1_LENGTH_SHIFT;
1425	val |= PORT_CS_1_WNR_WRITE;
1426	val |= (target << PORT_CS_1_TARGET_SHIFT) & PORT_CS_1_TARGET_MASK;
1427	if (target == USB4_SB_TARGET_RETIMER)
1428	val |= (index << PORT_CS_1_RETIMER_INDEX_SHIFT);
1429	val |= PORT_CS_1_PND;
1430
1431	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1432	offset: port->cap_usb4 + PORT_CS_1, length: 1);
1433	if (ret)
1434	return ret;
1435
1436	ret = usb4_port_wait_for_bit(port, offset: port->cap_usb4 + PORT_CS_1,
1437	PORT_CS_1_PND, value: 0, timeout_msec: 500, USB4_PORT_SB_DELAY);
1438	if (ret)
1439	return ret;
1440
1441	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1442	offset: port->cap_usb4 + PORT_CS_1, length: 1);
1443	if (ret)
1444	return ret;
1445
1446	if (val & PORT_CS_1_NR)
1447	return -ENODEV;
1448	if (val & PORT_CS_1_RC)
1449	return -EIO;
1450
1451	return 0;
1452	}
1453
1454	static int usb4_port_sb_opcode_err_to_errno(u32 val)
1455	{
1456	switch (val) {
1457	case 0:
1458	return 0;
1459	case USB4_SB_OPCODE_ERR:
1460	return -EAGAIN;
1461	case USB4_SB_OPCODE_ONS:
1462	return -EOPNOTSUPP;
1463	default:
1464	return -EIO;
1465	}
1466	}
1467
1468	static int usb4_port_sb_op(struct tb_port *port, enum usb4_sb_target target,
1469	u8 index, enum usb4_sb_opcode opcode, int timeout_msec)
1470	{
1471	ktime_t timeout;
1472	u32 val;
1473	int ret;
1474
1475	val = opcode;
1476	ret = usb4_port_sb_write(port, target, index, USB4_SB_OPCODE, buf: &val,
1477	size: sizeof(val));
1478	if (ret)
1479	return ret;
1480
1481	timeout = ktime_add_ms(kt: ktime_get(), msec: timeout_msec);
1482
1483	do {
1484	/* Check results */
1485	ret = usb4_port_sb_read(port, target, index, USB4_SB_OPCODE,
1486	buf: &val, size: sizeof(val));
1487	if (ret)
1488	return ret;
1489
1490	if (val != opcode)
1491	return usb4_port_sb_opcode_err_to_errno(val);
1492
1493	fsleep(USB4_PORT_SB_DELAY);
1494	} while (ktime_before(cmp1: ktime_get(), cmp2: timeout));
1495
1496	return -ETIMEDOUT;
1497	}
1498
1499	static int usb4_port_set_router_offline(struct tb_port *port, bool offline)
1500	{
1501	u32 val = !offline;
1502	int ret;
1503
1504	ret = usb4_port_sb_write(port, target: USB4_SB_TARGET_ROUTER, index: 0,
1505	USB4_SB_METADATA, buf: &val, size: sizeof(val));
1506	if (ret)
1507	return ret;
1508
1509	val = USB4_SB_OPCODE_ROUTER_OFFLINE;
1510	return usb4_port_sb_write(port, target: USB4_SB_TARGET_ROUTER, index: 0,
1511	USB4_SB_OPCODE, buf: &val, size: sizeof(val));
1512	}
1513
1514	/**
1515	* usb4_port_router_offline() - Put the USB4 port to offline mode
1516	* @port: USB4 port
1517	*
1518	* This function puts the USB4 port into offline mode. In this mode the
1519	* port does not react on hotplug events anymore. This needs to be
1520	* called before retimer access is done when the USB4 links is not up.
1521	*
1522	* Return: %0 on success, negative errno otherwise.
1523	*/
1524	int usb4_port_router_offline(struct tb_port *port)
1525	{
1526	return usb4_port_set_router_offline(port, offline: true);
1527	}
1528
1529	/**
1530	* usb4_port_router_online() - Put the USB4 port back online
1531	* @port: USB4 port
1532	*
1533	* Makes the USB4 port functional again.
1534	*
1535	* Return: %0 on success, negative errno otherwise.
1536	*/
1537	int usb4_port_router_online(struct tb_port *port)
1538	{
1539	return usb4_port_set_router_offline(port, offline: false);
1540	}
1541
1542	/**
1543	* usb4_port_enumerate_retimers() - Send RT broadcast transaction
1544	* @port: USB4 port
1545	*
1546	* This forces the USB4 port to send broadcast RT transaction which
1547	* makes the retimers on the link assign index to themselves.
1548	*
1549	* Return: %0 on success, negative errno otherwise.
1550	*/
1551	int usb4_port_enumerate_retimers(struct tb_port *port)
1552	{
1553	u32 val;
1554
1555	val = USB4_SB_OPCODE_ENUMERATE_RETIMERS;
1556	return usb4_port_sb_write(port, target: USB4_SB_TARGET_ROUTER, index: 0,
1557	USB4_SB_OPCODE, buf: &val, size: sizeof(val));
1558	}
1559
1560	/**
1561	* usb4_port_clx_supported() - Check if CLx is supported by the link
1562	* @port: Port to check for CLx support for
1563	*
1564	* PORT_CS_18_CPS bit reflects if the link supports CLx including
1565	* active cables (if connected on the link).
1566	*
1567	* Return: %true if Clx is supported, %false otherwise.
1568	*/
1569	bool usb4_port_clx_supported(struct tb_port *port)
1570	{
1571	int ret;
1572	u32 val;
1573
1574	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1575	offset: port->cap_usb4 + PORT_CS_18, length: 1);
1576	if (ret)
1577	return false;
1578
1579	return !!(val & PORT_CS_18_CPS);
1580	}
1581
1582	/**
1583	* usb4_port_asym_supported() - If the port supports asymmetric link
1584	* @port: USB4 port
1585	*
1586	* Checks if the port and the cable support asymmetric link.
1587	*
1588	* Return: %true if asymmetric link is supported, %false otherwise.
1589	*/
1590	bool usb4_port_asym_supported(struct tb_port *port)
1591	{
1592	u32 val;
1593
1594	if (!port->cap_usb4)
1595	return false;
1596
1597	if (tb_port_read(port, buffer: &val, space: TB_CFG_PORT, offset: port->cap_usb4 + PORT_CS_18, length: 1))
1598	return false;
1599
1600	return !!(val & PORT_CS_18_CSA);
1601	}
1602
1603	/**
1604	* usb4_port_asym_set_link_width() - Set link width to asymmetric or symmetric
1605	* @port: USB4 port
1606	* @width: Asymmetric width to configure
1607	*
1608	* Sets USB4 port link width to @width. Can be called for widths where
1609	* usb4_port_asym_width_supported() returned @true.
1610	*
1611	* Return: %0 on success, negative errno otherwise.
1612	*/
1613	int usb4_port_asym_set_link_width(struct tb_port *port, enum tb_link_width width)
1614	{
1615	u32 val;
1616	int ret;
1617
1618	if (!port->cap_phy)
1619	return -EINVAL;
1620
1621	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1622	offset: port->cap_phy + LANE_ADP_CS_1, length: 1);
1623	if (ret)
1624	return ret;
1625
1626	val &= ~LANE_ADP_CS_1_TARGET_WIDTH_ASYM_MASK;
1627	switch (width) {
1628	case TB_LINK_WIDTH_DUAL:
1629	val |= FIELD_PREP(LANE_ADP_CS_1_TARGET_WIDTH_ASYM_MASK,
1630	LANE_ADP_CS_1_TARGET_WIDTH_ASYM_DUAL);
1631	break;
1632	case TB_LINK_WIDTH_ASYM_TX:
1633	val |= FIELD_PREP(LANE_ADP_CS_1_TARGET_WIDTH_ASYM_MASK,
1634	LANE_ADP_CS_1_TARGET_WIDTH_ASYM_TX);
1635	break;
1636	case TB_LINK_WIDTH_ASYM_RX:
1637	val |= FIELD_PREP(LANE_ADP_CS_1_TARGET_WIDTH_ASYM_MASK,
1638	LANE_ADP_CS_1_TARGET_WIDTH_ASYM_RX);
1639	break;
1640	default:
1641	return -EINVAL;
1642	}
1643
1644	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1645	offset: port->cap_phy + LANE_ADP_CS_1, length: 1);
1646	}
1647
1648	/**
1649	* usb4_port_asym_start() - Start symmetry change and wait for completion
1650	* @port: USB4 port
1651	*
1652	* Start symmetry change of the link to asymmetric or symmetric
1653	* (according to what was previously set in tb_port_set_link_width().
1654	* Wait for completion of the change.
1655	*
1656	* Return:
1657	* * %0 - Symmetry change was successful.
1658	* * %-ETIMEDOUT - Timeout occurred.
1659	* * Negative errno - Other failure occurred.
1660	*/
1661	int usb4_port_asym_start(struct tb_port *port)
1662	{
1663	int ret;
1664	u32 val;
1665
1666	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
1667	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1668	if (ret)
1669	return ret;
1670
1671	val &= ~PORT_CS_19_START_ASYM;
1672	val |= FIELD_PREP(PORT_CS_19_START_ASYM, 1);
1673
1674	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
1675	offset: port->cap_usb4 + PORT_CS_19, length: 1);
1676	if (ret)
1677	return ret;
1678
1679	/*
1680	* Wait for PORT_CS_19_START_ASYM to be 0. This means the USB4
1681	* port started the symmetry transition.
1682	*/
1683	ret = usb4_port_wait_for_bit(port, offset: port->cap_usb4 + PORT_CS_19,
1684	PORT_CS_19_START_ASYM, value: 0, timeout_msec: 1000,
1685	USB4_PORT_DELAY);
1686	if (ret)
1687	return ret;
1688
1689	/* Then wait for the transtion to be completed */
1690	return usb4_port_wait_for_bit(port, offset: port->cap_usb4 + PORT_CS_18,
1691	PORT_CS_18_TIP, value: 0, timeout_msec: 5000, USB4_PORT_DELAY);
1692	}
1693
1694	/**
1695	* usb4_port_margining_caps() - Read USB4 port margining capabilities
1696	* @port: USB4 port
1697	* @target: Sideband target
1698	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1699	* @caps: Array with at least two elements to hold the results
1700	* @ncaps: Number of elements in the caps array
1701	*
1702	* Reads the USB4 port lane margining capabilities into @caps.
1703	*
1704	* Return: %0 on success, negative errno otherwise.
1705	*/
1706	int usb4_port_margining_caps(struct tb_port *port, enum usb4_sb_target target,
1707	u8 index, u32 *caps, size_t ncaps)
1708	{
1709	int ret;
1710
1711	ret = usb4_port_sb_op(port, target, index,
1712	opcode: USB4_SB_OPCODE_READ_LANE_MARGINING_CAP, timeout_msec: 500);
1713	if (ret)
1714	return ret;
1715
1716	return usb4_port_sb_read(port, target, index, USB4_SB_DATA, buf: caps,
1717	size: sizeof(*caps) * ncaps);
1718	}
1719
1720	/**
1721	* usb4_port_hw_margin() - Run hardware lane margining on port
1722	* @port: USB4 port
1723	* @target: Sideband target
1724	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1725	* @params: Parameters for USB4 hardware margining
1726	* @results: Array to hold the results
1727	* @nresults: Number of elements in the results array
1728	*
1729	* Runs hardware lane margining on USB4 port and returns the result in
1730	* @results.
1731	*
1732	* Return: %0 on success, negative errno otherwise.
1733	*/
1734	int usb4_port_hw_margin(struct tb_port *port, enum usb4_sb_target target,
1735	u8 index, const struct usb4_port_margining_params *params,
1736	u32 *results, size_t nresults)
1737	{
1738	u32 val;
1739	int ret;
1740
1741	if (WARN_ON_ONCE(!params))
1742	return -EINVAL;
1743
1744	val = params->lanes;
1745	if (params->time)
1746	val |= USB4_MARGIN_HW_TIME;
1747	if (params->right_high || params->upper_eye)
1748	val |= USB4_MARGIN_HW_RHU;
1749	if (params->ber_level)
1750	val |= FIELD_PREP(USB4_MARGIN_HW_BER_MASK, params->ber_level);
1751	if (params->optional_voltage_offset_range)
1752	val |= USB4_MARGIN_HW_OPT_VOLTAGE;
1753
1754	ret = usb4_port_sb_write(port, target, index, USB4_SB_METADATA, buf: &val,
1755	size: sizeof(val));
1756	if (ret)
1757	return ret;
1758
1759	ret = usb4_port_sb_op(port, target, index,
1760	opcode: USB4_SB_OPCODE_RUN_HW_LANE_MARGINING, timeout_msec: 2500);
1761	if (ret)
1762	return ret;
1763
1764	return usb4_port_sb_read(port, target, index, USB4_SB_DATA, buf: results,
1765	size: sizeof(*results) * nresults);
1766	}
1767
1768	/**
1769	* usb4_port_sw_margin() - Run software lane margining on port
1770	* @port: USB4 port
1771	* @target: Sideband target
1772	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1773	* @params: Parameters for USB4 software margining
1774	* @results: Data word for the operation completion data
1775	*
1776	* Runs software lane margining on USB4 port. Read back the error
1777	* counters by calling usb4_port_sw_margin_errors().
1778	*
1779	* Return: %0 on success, negative errno otherwise.
1780	*/
1781	int usb4_port_sw_margin(struct tb_port *port, enum usb4_sb_target target,
1782	u8 index, const struct usb4_port_margining_params *params,
1783	u32 *results)
1784	{
1785	u32 val;
1786	int ret;
1787
1788	if (WARN_ON_ONCE(!params))
1789	return -EINVAL;
1790
1791	val = params->lanes;
1792	if (params->time)
1793	val |= USB4_MARGIN_SW_TIME;
1794	if (params->optional_voltage_offset_range)
1795	val |= USB4_MARGIN_SW_OPT_VOLTAGE;
1796	if (params->right_high)
1797	val |= USB4_MARGIN_SW_RH;
1798	if (params->upper_eye)
1799	val |= USB4_MARGIN_SW_UPPER_EYE;
1800	val |= FIELD_PREP(USB4_MARGIN_SW_COUNTER_MASK, params->error_counter);
1801	val |= FIELD_PREP(USB4_MARGIN_SW_VT_MASK, params->voltage_time_offset);
1802
1803	ret = usb4_port_sb_write(port, target, index, USB4_SB_METADATA, buf: &val,
1804	size: sizeof(val));
1805	if (ret)
1806	return ret;
1807
1808	ret = usb4_port_sb_op(port, target, index,
1809	opcode: USB4_SB_OPCODE_RUN_SW_LANE_MARGINING, timeout_msec: 2500);
1810	if (ret)
1811	return ret;
1812
1813	return usb4_port_sb_read(port, target, index, USB4_SB_DATA, buf: results,
1814	size: sizeof(*results));
1815
1816	}
1817
1818	/**
1819	* usb4_port_sw_margin_errors() - Read the software margining error counters
1820	* @port: USB4 port
1821	* @target: Sideband target
1822	* @index: Retimer index if target is %USB4_SB_TARGET_RETIMER
1823	* @errors: Error metadata is copied here.
1824	*
1825	* This reads back the software margining error counters from the port.
1826	*
1827	* Return: %0 on success, negative errno otherwise.
1828	*/
1829	int usb4_port_sw_margin_errors(struct tb_port *port, enum usb4_sb_target target,
1830	u8 index, u32 *errors)
1831	{
1832	int ret;
1833
1834	ret = usb4_port_sb_op(port, target, index,
1835	opcode: USB4_SB_OPCODE_READ_SW_MARGIN_ERR, timeout_msec: 150);
1836	if (ret)
1837	return ret;
1838
1839	return usb4_port_sb_read(port, target, index, USB4_SB_METADATA, buf: errors,
1840	size: sizeof(*errors));
1841	}
1842
1843	static inline int usb4_port_retimer_op(struct tb_port *port, u8 index,
1844	enum usb4_sb_opcode opcode,
1845	int timeout_msec)
1846	{
1847	return usb4_port_sb_op(port, target: USB4_SB_TARGET_RETIMER, index, opcode,
1848	timeout_msec);
1849	}
1850
1851	/**
1852	* usb4_port_retimer_set_inbound_sbtx() - Enable sideband channel transactions
1853	* @port: USB4 port
1854	* @index: Retimer index
1855	*
1856	* Enables sideband channel transactions on SBTX. Can be used when USB4
1857	* link does not go up, for example if there is no device connected.
1858	*
1859	* Return: %0 on success, negative errno otherwise.
1860	*/
1861	int usb4_port_retimer_set_inbound_sbtx(struct tb_port *port, u8 index)
1862	{
1863	int ret;
1864
1865	ret = usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_SET_INBOUND_SBTX,
1866	timeout_msec: 500);
1867
1868	if (ret != -ENODEV)
1869	return ret;
1870
1871	/*
1872	* Per the USB4 retimer spec, the retimer is not required to
1873	* send an RT (Retimer Transaction) response for the first
1874	* SET_INBOUND_SBTX command
1875	*/
1876	return usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_SET_INBOUND_SBTX,
1877	timeout_msec: 500);
1878	}
1879
1880	/**
1881	* usb4_port_retimer_unset_inbound_sbtx() - Disable sideband channel transactions
1882	* @port: USB4 port
1883	* @index: Retimer index
1884	*
1885	* Disables sideband channel transactions on SBTX. The reverse of
1886	* usb4_port_retimer_set_inbound_sbtx().
1887	*
1888	* Return: %0 on success, negative errno otherwise.
1889	*/
1890	int usb4_port_retimer_unset_inbound_sbtx(struct tb_port *port, u8 index)
1891	{
1892	return usb4_port_retimer_op(port, index,
1893	opcode: USB4_SB_OPCODE_UNSET_INBOUND_SBTX, timeout_msec: 500);
1894	}
1895
1896	/**
1897	* usb4_port_retimer_is_last() - Is the retimer last on-board retimer
1898	* @port: USB4 port
1899	* @index: Retimer index
1900	*
1901	* Return:
1902	* * %1 - Retimer at @index is the last one (connected directly to the
1903	* Type-C port).
1904	* * %0 - Retimer at @index is not the last one.
1905	* * %-ENODEV - Retimer is not present.
1906	* * Negative errno - Other failure occurred.
1907	*/
1908	int usb4_port_retimer_is_last(struct tb_port *port, u8 index)
1909	{
1910	u32 metadata;
1911	int ret;
1912
1913	ret = usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_QUERY_LAST_RETIMER,
1914	timeout_msec: 500);
1915	if (ret)
1916	return ret;
1917
1918	ret = usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
1919	USB4_SB_METADATA, buf: &metadata, size: sizeof(metadata));
1920	return ret ? ret : metadata & 1;
1921	}
1922
1923	/**
1924	* usb4_port_retimer_is_cable() - Is the retimer cable retimer
1925	* @port: USB4 port
1926	* @index: Retimer index
1927	*
1928	* Return:
1929	* * %1 - Retimer at @index is the last cable retimer.
1930	* * %0 - Retimer at @index is on-board retimer.
1931	* * %-ENODEV - Retimer is not present.
1932	* * Negative errno - Other failure occurred.
1933	*/
1934	int usb4_port_retimer_is_cable(struct tb_port *port, u8 index)
1935	{
1936	u32 metadata;
1937	int ret;
1938
1939	ret = usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_QUERY_CABLE_RETIMER,
1940	timeout_msec: 500);
1941	if (ret)
1942	return ret;
1943
1944	ret = usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
1945	USB4_SB_METADATA, buf: &metadata, size: sizeof(metadata));
1946	return ret ? ret : metadata & 1;
1947	}
1948
1949	/**
1950	* usb4_port_retimer_nvm_sector_size() - Read retimer NVM sector size
1951	* @port: USB4 port
1952	* @index: Retimer index
1953	*
1954	* Reads NVM sector size (in bytes) of a retimer at @index. This
1955	* operation can be used to determine whether the retimer supports NVM
1956	* upgrade for example.
1957	*
1958	* Return:
1959	* * Sector size in bytes.
1960	* * %-ENODEV - If there is no retimer at @index.
1961	* * Negative errno - In case of an error.
1962	*/
1963	int usb4_port_retimer_nvm_sector_size(struct tb_port *port, u8 index)
1964	{
1965	u32 metadata;
1966	int ret;
1967
1968	ret = usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_GET_NVM_SECTOR_SIZE,
1969	timeout_msec: 500);
1970	if (ret)
1971	return ret;
1972
1973	ret = usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
1974	USB4_SB_METADATA, buf: &metadata, size: sizeof(metadata));
1975	return ret ? ret : metadata & USB4_NVM_SECTOR_SIZE_MASK;
1976	}
1977
1978	/**
1979	* usb4_port_retimer_nvm_set_offset() - Set NVM write offset
1980	* @port: USB4 port
1981	* @index: Retimer index
1982	* @address: Start offset
1983	*
1984	* Explicitly sets NVM write offset. Normally when writing to NVM this is
1985	* done automatically by usb4_port_retimer_nvm_write().
1986	*
1987	* Return: %0 on success, negative errno otherwise.
1988	*/
1989	int usb4_port_retimer_nvm_set_offset(struct tb_port *port, u8 index,
1990	unsigned int address)
1991	{
1992	u32 metadata, dwaddress;
1993	int ret;
1994
1995	dwaddress = address / 4;
1996	metadata = (dwaddress << USB4_NVM_SET_OFFSET_SHIFT) &
1997	USB4_NVM_SET_OFFSET_MASK;
1998
1999	ret = usb4_port_sb_write(port, target: USB4_SB_TARGET_RETIMER, index,
2000	USB4_SB_METADATA, buf: &metadata, size: sizeof(metadata));
2001	if (ret)
2002	return ret;
2003
2004	return usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_NVM_SET_OFFSET,
2005	timeout_msec: 500);
2006	}
2007
2008	struct retimer_info {
2009	struct tb_port *port;
2010	u8 index;
2011	};
2012
2013	static int usb4_port_retimer_nvm_write_next_block(void *data,
2014	unsigned int dwaddress, const void *buf, size_t dwords)
2015
2016	{
2017	const struct retimer_info *info = data;
2018	struct tb_port *port = info->port;
2019	u8 index = info->index;
2020	int ret;
2021
2022	ret = usb4_port_sb_write(port, target: USB4_SB_TARGET_RETIMER, index,
2023	USB4_SB_DATA, buf, size: dwords * 4);
2024	if (ret)
2025	return ret;
2026
2027	return usb4_port_retimer_op(port, index,
2028	opcode: USB4_SB_OPCODE_NVM_BLOCK_WRITE, timeout_msec: 1000);
2029	}
2030
2031	/**
2032	* usb4_port_retimer_nvm_write() - Write to retimer NVM
2033	* @port: USB4 port
2034	* @index: Retimer index
2035	* @address: Byte address where to start the write
2036	* @buf: Data to write
2037	* @size: Size in bytes how much to write
2038	*
2039	* Writes @size bytes from @buf to the retimer NVM. Used for NVM
2040	* upgrade.
2041	*
2042	* Return:
2043	* * %0 - If the data was written successfully.
2044	* * %-ENODEV - If there is no retimer at @index.
2045	* * Negative errno - In case of an error.
2046	*/
2047	int usb4_port_retimer_nvm_write(struct tb_port *port, u8 index, unsigned int address,
2048	const void *buf, size_t size)
2049	{
2050	struct retimer_info info = { .port = port, .index = index };
2051	int ret;
2052
2053	ret = usb4_port_retimer_nvm_set_offset(port, index, address);
2054	if (ret)
2055	return ret;
2056
2057	return tb_nvm_write_data(address, buf, size, USB4_DATA_RETRIES,
2058	write_next_block: usb4_port_retimer_nvm_write_next_block, write_block_data: &info);
2059	}
2060
2061	/**
2062	* usb4_port_retimer_nvm_authenticate() - Start retimer NVM upgrade
2063	* @port: USB4 port
2064	* @index: Retimer index
2065	*
2066	* After the new NVM image has been written via usb4_port_retimer_nvm_write()
2067	* this function can be used to trigger the NVM upgrade process. If
2068	* successful the retimer restarts with the new NVM and may not have the
2069	* index set so one needs to call usb4_port_enumerate_retimers() to
2070	* force index to be assigned.
2071	*
2072	* Return: %0 on success, negative errno otherwise.
2073	*/
2074	int usb4_port_retimer_nvm_authenticate(struct tb_port *port, u8 index)
2075	{
2076	u32 val;
2077
2078	/*
2079	* We need to use the raw operation here because once the
2080	* authentication completes the retimer index is not set anymore
2081	* so we do not get back the status now.
2082	*/
2083	val = USB4_SB_OPCODE_NVM_AUTH_WRITE;
2084	return usb4_port_sb_write(port, target: USB4_SB_TARGET_RETIMER, index,
2085	USB4_SB_OPCODE, buf: &val, size: sizeof(val));
2086	}
2087
2088	/**
2089	* usb4_port_retimer_nvm_authenticate_status() - Read status of NVM upgrade
2090	* @port: USB4 port
2091	* @index: Retimer index
2092	* @status: Raw status code read from metadata
2093	*
2094	* This can be called after usb4_port_retimer_nvm_authenticate() and
2095	* usb4_port_enumerate_retimers() to fetch status of the NVM upgrade.
2096	*
2097	* Return: %0 if the authentication status was successfully read. The
2098	* completion metadata (the result) is then stored into @status. If
2099	* status read fails, returns negative errno.
2100	*/
2101	int usb4_port_retimer_nvm_authenticate_status(struct tb_port *port, u8 index,
2102	u32 *status)
2103	{
2104	u32 metadata, val;
2105	int ret;
2106
2107	ret = usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
2108	USB4_SB_OPCODE, buf: &val, size: sizeof(val));
2109	if (ret)
2110	return ret;
2111
2112	ret = usb4_port_sb_opcode_err_to_errno(val);
2113	switch (ret) {
2114	case 0:
2115	*status = 0;
2116	return 0;
2117
2118	case -EAGAIN:
2119	ret = usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
2120	USB4_SB_METADATA, buf: &metadata,
2121	size: sizeof(metadata));
2122	if (ret)
2123	return ret;
2124
2125	*status = metadata & USB4_SB_METADATA_NVM_AUTH_WRITE_MASK;
2126	return 0;
2127
2128	default:
2129	return ret;
2130	}
2131	}
2132
2133	static int usb4_port_retimer_nvm_read_block(void *data, unsigned int dwaddress,
2134	void *buf, size_t dwords)
2135	{
2136	const struct retimer_info *info = data;
2137	struct tb_port *port = info->port;
2138	u8 index = info->index;
2139	u32 metadata;
2140	int ret;
2141
2142	metadata = dwaddress << USB4_NVM_READ_OFFSET_SHIFT;
2143	if (dwords < USB4_DATA_DWORDS)
2144	metadata |= dwords << USB4_NVM_READ_LENGTH_SHIFT;
2145
2146	ret = usb4_port_sb_write(port, target: USB4_SB_TARGET_RETIMER, index,
2147	USB4_SB_METADATA, buf: &metadata, size: sizeof(metadata));
2148	if (ret)
2149	return ret;
2150
2151	ret = usb4_port_retimer_op(port, index, opcode: USB4_SB_OPCODE_NVM_READ, timeout_msec: 500);
2152	if (ret)
2153	return ret;
2154
2155	return usb4_port_sb_read(port, target: USB4_SB_TARGET_RETIMER, index,
2156	USB4_SB_DATA, buf, size: dwords * 4);
2157	}
2158
2159	/**
2160	* usb4_port_retimer_nvm_read() - Read contents of retimer NVM
2161	* @port: USB4 port
2162	* @index: Retimer index
2163	* @address: NVM address (in bytes) to start reading
2164	* @buf: Data read from NVM is stored here
2165	* @size: Number of bytes to read
2166	*
2167	* Reads retimer NVM and copies the contents to @buf.
2168	*
2169	* Return:
2170	* * %0 - If the read was successful.
2171	* * %-ENODEV - If there is no retimer at @index.
2172	* * Negative errno - In case of an error.
2173	*/
2174	int usb4_port_retimer_nvm_read(struct tb_port *port, u8 index,
2175	unsigned int address, void *buf, size_t size)
2176	{
2177	struct retimer_info info = { .port = port, .index = index };
2178
2179	return tb_nvm_read_data(address, buf, size, USB4_DATA_RETRIES,
2180	read_block: usb4_port_retimer_nvm_read_block, read_block_data: &info);
2181	}
2182
2183	static inline unsigned int
2184	usb4_usb3_port_max_bandwidth(const struct tb_port *port, unsigned int bw)
2185	{
2186	/* Take the possible bandwidth limitation into account */
2187	if (port->max_bw)
2188	return min(bw, port->max_bw);
2189	return bw;
2190	}
2191
2192	/**
2193	* usb4_usb3_port_max_link_rate() - Maximum supported USB3 link rate
2194	* @port: USB3 adapter port
2195	*
2196	* Return: Maximum supported link rate of a USB3 adapter in Mb/s.
2197	* Negative errno in case of an error.
2198	*/
2199	int usb4_usb3_port_max_link_rate(struct tb_port *port)
2200	{
2201	int ret, lr;
2202	u32 val;
2203
2204	if (!tb_port_is_usb3_down(port) && !tb_port_is_usb3_up(port))
2205	return -EINVAL;
2206
2207	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2208	offset: port->cap_adap + ADP_USB3_CS_4, length: 1);
2209	if (ret)
2210	return ret;
2211
2212	lr = (val & ADP_USB3_CS_4_MSLR_MASK) >> ADP_USB3_CS_4_MSLR_SHIFT;
2213	ret = lr == ADP_USB3_CS_4_MSLR_20G ? 20000 : 10000;
2214
2215	return usb4_usb3_port_max_bandwidth(port, bw: ret);
2216	}
2217
2218	static int usb4_usb3_port_cm_request(struct tb_port *port, bool request)
2219	{
2220	int ret;
2221	u32 val;
2222
2223	if (!tb_port_is_usb3_down(port))
2224	return -EINVAL;
2225	if (tb_route(sw: port->sw))
2226	return -EINVAL;
2227
2228	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2229	offset: port->cap_adap + ADP_USB3_CS_2, length: 1);
2230	if (ret)
2231	return ret;
2232
2233	if (request)
2234	val |= ADP_USB3_CS_2_CMR;
2235	else
2236	val &= ~ADP_USB3_CS_2_CMR;
2237
2238	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2239	offset: port->cap_adap + ADP_USB3_CS_2, length: 1);
2240	if (ret)
2241	return ret;
2242
2243	/*
2244	* We can use val here directly as the CMR bit is in the same place
2245	* as HCA. Just mask out others.
2246	*/
2247	val &= ADP_USB3_CS_2_CMR;
2248	return usb4_port_wait_for_bit(port, offset: port->cap_adap + ADP_USB3_CS_1,
2249	ADP_USB3_CS_1_HCA, value: val, timeout_msec: 1500,
2250	USB4_PORT_DELAY);
2251	}
2252
2253	static inline int usb4_usb3_port_set_cm_request(struct tb_port *port)
2254	{
2255	return usb4_usb3_port_cm_request(port, request: true);
2256	}
2257
2258	static inline int usb4_usb3_port_clear_cm_request(struct tb_port *port)
2259	{
2260	return usb4_usb3_port_cm_request(port, request: false);
2261	}
2262
2263	static unsigned int usb3_bw_to_mbps(u32 bw, u8 scale)
2264	{
2265	unsigned long uframes;
2266
2267	uframes = bw * 512UL << scale;
2268	return DIV_ROUND_CLOSEST(uframes * 8000, MEGA);
2269	}
2270
2271	static u32 mbps_to_usb3_bw(unsigned int mbps, u8 scale)
2272	{
2273	unsigned long uframes;
2274
2275	/* 1 uframe is 1/8 ms (125 us) -> 1 / 8000 s */
2276	uframes = ((unsigned long)mbps * MEGA) / 8000;
2277	return DIV_ROUND_UP(uframes, 512UL << scale);
2278	}
2279
2280	static int usb4_usb3_port_read_allocated_bandwidth(struct tb_port *port,
2281	int *upstream_bw,
2282	int *downstream_bw)
2283	{
2284	u32 val, bw, scale;
2285	int ret;
2286
2287	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2288	offset: port->cap_adap + ADP_USB3_CS_2, length: 1);
2289	if (ret)
2290	return ret;
2291
2292	ret = tb_port_read(port, buffer: &scale, space: TB_CFG_PORT,
2293	offset: port->cap_adap + ADP_USB3_CS_3, length: 1);
2294	if (ret)
2295	return ret;
2296
2297	scale &= ADP_USB3_CS_3_SCALE_MASK;
2298
2299	bw = val & ADP_USB3_CS_2_AUBW_MASK;
2300	*upstream_bw = usb3_bw_to_mbps(bw, scale);
2301
2302	bw = (val & ADP_USB3_CS_2_ADBW_MASK) >> ADP_USB3_CS_2_ADBW_SHIFT;
2303	*downstream_bw = usb3_bw_to_mbps(bw, scale);
2304
2305	return 0;
2306	}
2307
2308	/**
2309	* usb4_usb3_port_allocated_bandwidth() - Bandwidth allocated for USB3
2310	* @port: USB3 adapter port
2311	* @upstream_bw: Allocated upstream bandwidth is stored here
2312	* @downstream_bw: Allocated downstream bandwidth is stored here
2313	*
2314	* Stores currently allocated USB3 bandwidth into @upstream_bw and
2315	* @downstream_bw in Mb/s.
2316	*
2317	* Return: %0 on success, negative errno otherwise.
2318	*/
2319	int usb4_usb3_port_allocated_bandwidth(struct tb_port *port, int *upstream_bw,
2320	int *downstream_bw)
2321	{
2322	int ret;
2323
2324	ret = usb4_usb3_port_set_cm_request(port);
2325	if (ret)
2326	return ret;
2327
2328	ret = usb4_usb3_port_read_allocated_bandwidth(port, upstream_bw,
2329	downstream_bw);
2330	usb4_usb3_port_clear_cm_request(port);
2331
2332	return ret;
2333	}
2334
2335	static int usb4_usb3_port_read_consumed_bandwidth(struct tb_port *port,
2336	int *upstream_bw,
2337	int *downstream_bw)
2338	{
2339	u32 val, bw, scale;
2340	int ret;
2341
2342	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2343	offset: port->cap_adap + ADP_USB3_CS_1, length: 1);
2344	if (ret)
2345	return ret;
2346
2347	ret = tb_port_read(port, buffer: &scale, space: TB_CFG_PORT,
2348	offset: port->cap_adap + ADP_USB3_CS_3, length: 1);
2349	if (ret)
2350	return ret;
2351
2352	scale &= ADP_USB3_CS_3_SCALE_MASK;
2353
2354	bw = val & ADP_USB3_CS_1_CUBW_MASK;
2355	*upstream_bw = usb3_bw_to_mbps(bw, scale);
2356
2357	bw = (val & ADP_USB3_CS_1_CDBW_MASK) >> ADP_USB3_CS_1_CDBW_SHIFT;
2358	*downstream_bw = usb3_bw_to_mbps(bw, scale);
2359
2360	return 0;
2361	}
2362
2363	static int usb4_usb3_port_write_allocated_bandwidth(struct tb_port *port,
2364	int upstream_bw,
2365	int downstream_bw)
2366	{
2367	u32 val, ubw, dbw, scale;
2368	int ret, max_bw;
2369
2370	/* Figure out suitable scale */
2371	scale = 0;
2372	max_bw = max(upstream_bw, downstream_bw);
2373	while (scale < 64) {
2374	if (mbps_to_usb3_bw(mbps: max_bw, scale) < 4096)
2375	break;
2376	scale++;
2377	}
2378
2379	if (WARN_ON(scale >= 64))
2380	return -EINVAL;
2381
2382	ret = tb_port_write(port, buffer: &scale, space: TB_CFG_PORT,
2383	offset: port->cap_adap + ADP_USB3_CS_3, length: 1);
2384	if (ret)
2385	return ret;
2386
2387	ubw = mbps_to_usb3_bw(mbps: upstream_bw, scale);
2388	dbw = mbps_to_usb3_bw(mbps: downstream_bw, scale);
2389
2390	tb_port_dbg(port, "scaled bandwidth %u/%u, scale %u\n", ubw, dbw, scale);
2391
2392	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2393	offset: port->cap_adap + ADP_USB3_CS_2, length: 1);
2394	if (ret)
2395	return ret;
2396
2397	val &= ~(ADP_USB3_CS_2_AUBW_MASK | ADP_USB3_CS_2_ADBW_MASK);
2398	val |= dbw << ADP_USB3_CS_2_ADBW_SHIFT;
2399	val |= ubw;
2400
2401	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2402	offset: port->cap_adap + ADP_USB3_CS_2, length: 1);
2403	}
2404
2405	/**
2406	* usb4_usb3_port_allocate_bandwidth() - Allocate bandwidth for USB3
2407	* @port: USB3 adapter port
2408	* @upstream_bw: New upstream bandwidth
2409	* @downstream_bw: New downstream bandwidth
2410	*
2411	* This can be used to set how much bandwidth is allocated for the USB3
2412	* tunneled isochronous traffic. @upstream_bw and @downstream_bw are the
2413	* new values programmed to the USB3 adapter allocation registers. If
2414	* the values are lower than what is currently consumed the allocation
2415	* is set to what is currently consumed instead (consumed bandwidth
2416	* cannot be taken away by CM). The actual new values are returned in
2417	* @upstream_bw and @downstream_bw.
2418	*
2419	* Return: %0 on success, negative errno otherwise.
2420	*/
2421	int usb4_usb3_port_allocate_bandwidth(struct tb_port *port, int *upstream_bw,
2422	int *downstream_bw)
2423	{
2424	int ret, consumed_up, consumed_down, allocate_up, allocate_down;
2425
2426	ret = usb4_usb3_port_set_cm_request(port);
2427	if (ret)
2428	return ret;
2429
2430	ret = usb4_usb3_port_read_consumed_bandwidth(port, upstream_bw: &consumed_up,
2431	downstream_bw: &consumed_down);
2432	if (ret)
2433	goto err_request;
2434
2435	/* Don't allow it go lower than what is consumed */
2436	allocate_up = max(*upstream_bw, consumed_up);
2437	allocate_down = max(*downstream_bw, consumed_down);
2438
2439	ret = usb4_usb3_port_write_allocated_bandwidth(port, upstream_bw: allocate_up,
2440	downstream_bw: allocate_down);
2441	if (ret)
2442	goto err_request;
2443
2444	*upstream_bw = allocate_up;
2445	*downstream_bw = allocate_down;
2446
2447	err_request:
2448	usb4_usb3_port_clear_cm_request(port);
2449	return ret;
2450	}
2451
2452	/**
2453	* usb4_usb3_port_release_bandwidth() - Release allocated USB3 bandwidth
2454	* @port: USB3 adapter port
2455	* @upstream_bw: New allocated upstream bandwidth
2456	* @downstream_bw: New allocated downstream bandwidth
2457	*
2458	* Releases USB3 allocated bandwidth down to what is actually consumed.
2459	* The new bandwidth is returned in @upstream_bw and @downstream_bw.
2460	*
2461	* Return: %0 on success, negative errno otherwise.
2462	*/
2463	int usb4_usb3_port_release_bandwidth(struct tb_port *port, int *upstream_bw,
2464	int *downstream_bw)
2465	{
2466	int ret, consumed_up, consumed_down;
2467
2468	ret = usb4_usb3_port_set_cm_request(port);
2469	if (ret)
2470	return ret;
2471
2472	ret = usb4_usb3_port_read_consumed_bandwidth(port, upstream_bw: &consumed_up,
2473	downstream_bw: &consumed_down);
2474	if (ret)
2475	goto err_request;
2476
2477	/*
2478	* Always keep 900 Mb/s to make sure xHCI has at least some
2479	* bandwidth available for isochronous traffic.
2480	*/
2481	if (consumed_up < 900)
2482	consumed_up = 900;
2483	if (consumed_down < 900)
2484	consumed_down = 900;
2485
2486	ret = usb4_usb3_port_write_allocated_bandwidth(port, upstream_bw: consumed_up,
2487	downstream_bw: consumed_down);
2488	if (ret)
2489	goto err_request;
2490
2491	*upstream_bw = consumed_up;
2492	*downstream_bw = consumed_down;
2493
2494	err_request:
2495	usb4_usb3_port_clear_cm_request(port);
2496	return ret;
2497	}
2498
2499	static bool is_usb4_dpin(const struct tb_port *port)
2500	{
2501	if (!tb_port_is_dpin(port))
2502	return false;
2503	if (!tb_switch_is_usb4(sw: port->sw))
2504	return false;
2505	return true;
2506	}
2507
2508	/**
2509	* usb4_dp_port_set_cm_id() - Assign CM ID to the DP IN adapter
2510	* @port: DP IN adapter
2511	* @cm_id: CM ID to assign
2512	*
2513	* Sets CM ID for the @port.
2514	*
2515	* Return:
2516	* * %0 - On success.
2517	* * %-EOPNOTSUPP - If the @port does not support this.
2518	* * Negative errno - Another error occurred.
2519	*/
2520	int usb4_dp_port_set_cm_id(struct tb_port *port, int cm_id)
2521	{
2522	u32 val;
2523	int ret;
2524
2525	if (!is_usb4_dpin(port))
2526	return -EOPNOTSUPP;
2527
2528	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2529	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2530	if (ret)
2531	return ret;
2532
2533	val &= ~ADP_DP_CS_2_CM_ID_MASK;
2534	val |= cm_id << ADP_DP_CS_2_CM_ID_SHIFT;
2535
2536	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2537	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2538	}
2539
2540	/**
2541	* usb4_dp_port_bandwidth_mode_supported() - Is the bandwidth allocation mode
2542	* supported
2543	* @port: DP IN adapter to check
2544	*
2545	* Can be called to any DP IN adapter.
2546	*
2547	* Return: %true if the adapter supports USB4 bandwidth allocation mode,
2548	* %false otherwise.
2549	*/
2550	bool usb4_dp_port_bandwidth_mode_supported(struct tb_port *port)
2551	{
2552	int ret;
2553	u32 val;
2554
2555	if (!is_usb4_dpin(port))
2556	return false;
2557
2558	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2559	offset: port->cap_adap + DP_LOCAL_CAP, length: 1);
2560	if (ret)
2561	return false;
2562
2563	return !!(val & DP_COMMON_CAP_BW_MODE);
2564	}
2565
2566	/**
2567	* usb4_dp_port_bandwidth_mode_enabled() - Is the bandwidth allocation mode
2568	* enabled
2569	* @port: DP IN adapter to check
2570	*
2571	* Can be called to any DP IN adapter.
2572	*
2573	* Return: %true if the bandwidth allocation mode has been enabled,
2574	* %false otherwise.
2575	*/
2576	bool usb4_dp_port_bandwidth_mode_enabled(struct tb_port *port)
2577	{
2578	int ret;
2579	u32 val;
2580
2581	if (!is_usb4_dpin(port))
2582	return false;
2583
2584	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2585	offset: port->cap_adap + ADP_DP_CS_8, length: 1);
2586	if (ret)
2587	return false;
2588
2589	return !!(val & ADP_DP_CS_8_DPME);
2590	}
2591
2592	/**
2593	* usb4_dp_port_set_cm_bandwidth_mode_supported() - Set/clear CM support for
2594	* bandwidth allocation mode
2595	* @port: DP IN adapter
2596	* @supported: Does the CM support bandwidth allocation mode
2597	*
2598	* Can be called to any DP IN adapter. Sets or clears the CM support bit
2599	* of the DP IN adapter.
2600	*
2601	* * Return:
2602	* * %0 - On success.
2603	* * %-EOPNOTSUPP - If the passed IN adapter does not support this.
2604	* * Negative errno - Another error occurred.
2605	*/
2606	int usb4_dp_port_set_cm_bandwidth_mode_supported(struct tb_port *port,
2607	bool supported)
2608	{
2609	u32 val;
2610	int ret;
2611
2612	if (!is_usb4_dpin(port))
2613	return -EOPNOTSUPP;
2614
2615	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2616	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2617	if (ret)
2618	return ret;
2619
2620	if (supported)
2621	val |= ADP_DP_CS_2_CMMS;
2622	else
2623	val &= ~ADP_DP_CS_2_CMMS;
2624
2625	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2626	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2627	}
2628
2629	/**
2630	* usb4_dp_port_group_id() - Return Group ID assigned for the adapter
2631	* @port: DP IN adapter
2632	*
2633	* Reads bandwidth allocation Group ID from the DP IN adapter and
2634	* returns it.
2635	*
2636	* Return:
2637	* * Group ID assigned to adapter @port.
2638	* * %-EOPNOTSUPP - If adapter does not support setting GROUP_ID.
2639	* * Negative errno - Another error occurred.
2640	*/
2641	int usb4_dp_port_group_id(struct tb_port *port)
2642	{
2643	u32 val;
2644	int ret;
2645
2646	if (!is_usb4_dpin(port))
2647	return -EOPNOTSUPP;
2648
2649	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2650	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2651	if (ret)
2652	return ret;
2653
2654	return (val & ADP_DP_CS_2_GROUP_ID_MASK) >> ADP_DP_CS_2_GROUP_ID_SHIFT;
2655	}
2656
2657	/**
2658	* usb4_dp_port_set_group_id() - Set adapter Group ID
2659	* @port: DP IN adapter
2660	* @group_id: Group ID for the adapter
2661	*
2662	* Sets bandwidth allocation mode Group ID for the DP IN adapter.
2663	*
2664	* Return:
2665	* * %0 - On success.
2666	* * %-EOPNOTSUPP - If the adapter does not support this.
2667	* * Negative errno - Another error occurred.
2668	*/
2669	int usb4_dp_port_set_group_id(struct tb_port *port, int group_id)
2670	{
2671	u32 val;
2672	int ret;
2673
2674	if (!is_usb4_dpin(port))
2675	return -EOPNOTSUPP;
2676
2677	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2678	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2679	if (ret)
2680	return ret;
2681
2682	val &= ~ADP_DP_CS_2_GROUP_ID_MASK;
2683	val |= group_id << ADP_DP_CS_2_GROUP_ID_SHIFT;
2684
2685	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2686	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2687	}
2688
2689	/**
2690	* usb4_dp_port_nrd() - Read non-reduced rate and lanes
2691	* @port: DP IN adapter
2692	* @rate: Non-reduced rate in Mb/s is placed here
2693	* @lanes: Non-reduced lanes are placed here
2694	*
2695	* Reads the non-reduced rate and lanes from the DP IN adapter.
2696	*
2697	* Return:
2698	* * %0 - On success.
2699	* * %-EOPNOTSUPP - If the adapter does not support this.
2700	* * Negative errno - Another error occurred.
2701	*/
2702	int usb4_dp_port_nrd(struct tb_port *port, int *rate, int *lanes)
2703	{
2704	u32 val, tmp;
2705	int ret;
2706
2707	if (!is_usb4_dpin(port))
2708	return -EOPNOTSUPP;
2709
2710	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2711	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2712	if (ret)
2713	return ret;
2714
2715	tmp = (val & ADP_DP_CS_2_NRD_MLR_MASK) >> ADP_DP_CS_2_NRD_MLR_SHIFT;
2716	switch (tmp) {
2717	case DP_COMMON_CAP_RATE_RBR:
2718	*rate = 1620;
2719	break;
2720	case DP_COMMON_CAP_RATE_HBR:
2721	*rate = 2700;
2722	break;
2723	case DP_COMMON_CAP_RATE_HBR2:
2724	*rate = 5400;
2725	break;
2726	case DP_COMMON_CAP_RATE_HBR3:
2727	*rate = 8100;
2728	break;
2729	}
2730
2731	tmp = val & ADP_DP_CS_2_NRD_MLC_MASK;
2732	switch (tmp) {
2733	case DP_COMMON_CAP_1_LANE:
2734	*lanes = 1;
2735	break;
2736	case DP_COMMON_CAP_2_LANES:
2737	*lanes = 2;
2738	break;
2739	case DP_COMMON_CAP_4_LANES:
2740	*lanes = 4;
2741	break;
2742	}
2743
2744	return 0;
2745	}
2746
2747	/**
2748	* usb4_dp_port_set_nrd() - Set non-reduced rate and lanes
2749	* @port: DP IN adapter
2750	* @rate: Non-reduced rate in Mb/s
2751	* @lanes: Non-reduced lanes
2752	*
2753	* Before the capabilities reduction, this function can be used to set
2754	* the non-reduced values for the DP IN adapter.
2755	*
2756	* Return:
2757	* * %0 - On success.
2758	* * %-EOPNOTSUPP - If the adapter does not support this.
2759	* * Negative errno - Another error occurred.
2760	*/
2761	int usb4_dp_port_set_nrd(struct tb_port *port, int rate, int lanes)
2762	{
2763	u32 val;
2764	int ret;
2765
2766	if (!is_usb4_dpin(port))
2767	return -EOPNOTSUPP;
2768
2769	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2770	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2771	if (ret)
2772	return ret;
2773
2774	val &= ~ADP_DP_CS_2_NRD_MLR_MASK;
2775
2776	switch (rate) {
2777	case 1620:
2778	break;
2779	case 2700:
2780	val |= (DP_COMMON_CAP_RATE_HBR << ADP_DP_CS_2_NRD_MLR_SHIFT)
2781	& ADP_DP_CS_2_NRD_MLR_MASK;
2782	break;
2783	case 5400:
2784	val |= (DP_COMMON_CAP_RATE_HBR2 << ADP_DP_CS_2_NRD_MLR_SHIFT)
2785	& ADP_DP_CS_2_NRD_MLR_MASK;
2786	break;
2787	case 8100:
2788	val |= (DP_COMMON_CAP_RATE_HBR3 << ADP_DP_CS_2_NRD_MLR_SHIFT)
2789	& ADP_DP_CS_2_NRD_MLR_MASK;
2790	break;
2791	default:
2792	return -EINVAL;
2793	}
2794
2795	val &= ~ADP_DP_CS_2_NRD_MLC_MASK;
2796
2797	switch (lanes) {
2798	case 1:
2799	break;
2800	case 2:
2801	val |= DP_COMMON_CAP_2_LANES;
2802	break;
2803	case 4:
2804	val |= DP_COMMON_CAP_4_LANES;
2805	break;
2806	default:
2807	return -EINVAL;
2808	}
2809
2810	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2811	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2812	}
2813
2814	/**
2815	* usb4_dp_port_granularity() - Return granularity for the bandwidth values
2816	* @port: DP IN adapter
2817	*
2818	* Reads the programmed granularity from @port.
2819	*
2820	* Return:
2821	* * Granularity value of a @port.
2822	* * %-EOPNOTSUPP - If the DP IN adapter does not support bandwidth
2823	* allocation mode.
2824	* * Negative errno - Another error occurred.
2825	*/
2826	int usb4_dp_port_granularity(struct tb_port *port)
2827	{
2828	u32 val;
2829	int ret;
2830
2831	if (!is_usb4_dpin(port))
2832	return -EOPNOTSUPP;
2833
2834	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2835	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2836	if (ret)
2837	return ret;
2838
2839	val &= ADP_DP_CS_2_GR_MASK;
2840	val >>= ADP_DP_CS_2_GR_SHIFT;
2841
2842	switch (val) {
2843	case ADP_DP_CS_2_GR_0_25G:
2844	return 250;
2845	case ADP_DP_CS_2_GR_0_5G:
2846	return 500;
2847	case ADP_DP_CS_2_GR_1G:
2848	return 1000;
2849	}
2850
2851	return -EINVAL;
2852	}
2853
2854	/**
2855	* usb4_dp_port_set_granularity() - Set granularity for the bandwidth values
2856	* @port: DP IN adapter
2857	* @granularity: Granularity in Mb/s. Supported values: 1000, 500 and 250.
2858	*
2859	* Sets the granularity used with the estimated, allocated and requested
2860	* bandwidth.
2861	*
2862	* Return:
2863	* * %0 - On success.
2864	* * %-EOPNOTSUPP - If the adapter does not support this.
2865	* * Negative errno - Another error occurred.
2866	*/
2867	int usb4_dp_port_set_granularity(struct tb_port *port, int granularity)
2868	{
2869	u32 val;
2870	int ret;
2871
2872	if (!is_usb4_dpin(port))
2873	return -EOPNOTSUPP;
2874
2875	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2876	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2877	if (ret)
2878	return ret;
2879
2880	val &= ~ADP_DP_CS_2_GR_MASK;
2881
2882	switch (granularity) {
2883	case 250:
2884	val |= ADP_DP_CS_2_GR_0_25G << ADP_DP_CS_2_GR_SHIFT;
2885	break;
2886	case 500:
2887	val |= ADP_DP_CS_2_GR_0_5G << ADP_DP_CS_2_GR_SHIFT;
2888	break;
2889	case 1000:
2890	val |= ADP_DP_CS_2_GR_1G << ADP_DP_CS_2_GR_SHIFT;
2891	break;
2892	default:
2893	return -EINVAL;
2894	}
2895
2896	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2897	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2898	}
2899
2900	/**
2901	* usb4_dp_port_set_estimated_bandwidth() - Set estimated bandwidth
2902	* @port: DP IN adapter
2903	* @bw: Estimated bandwidth in Mb/s.
2904	*
2905	* Sets the estimated bandwidth to @bw. Set the granularity by calling
2906	* usb4_dp_port_set_granularity() before calling this. The @bw is rounded
2907	* down to the closest granularity multiplier.
2908	*
2909	* Return:
2910	* * %0 - On success.
2911	* * %-EOPNOTSUPP - If the adapter does not support this.
2912	* * Negative errno - Another error occurred.
2913	*/
2914	int usb4_dp_port_set_estimated_bandwidth(struct tb_port *port, int bw)
2915	{
2916	u32 val, granularity;
2917	int ret;
2918
2919	if (!is_usb4_dpin(port))
2920	return -EOPNOTSUPP;
2921
2922	ret = usb4_dp_port_granularity(port);
2923	if (ret < 0)
2924	return ret;
2925	granularity = ret;
2926
2927	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2928	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2929	if (ret)
2930	return ret;
2931
2932	val &= ~ADP_DP_CS_2_ESTIMATED_BW_MASK;
2933	val |= (bw / granularity) << ADP_DP_CS_2_ESTIMATED_BW_SHIFT;
2934
2935	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2936	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2937	}
2938
2939	/**
2940	* usb4_dp_port_allocated_bandwidth() - Return allocated bandwidth
2941	* @port: DP IN adapter
2942	*
2943	* Reads the allocated bandwidth for @port in Mb/s (taking into account
2944	* the programmed granularity).
2945	*
2946	* Return: Allocated bandwidth in Mb/s or negative errno in case of an error.
2947	*/
2948	int usb4_dp_port_allocated_bandwidth(struct tb_port *port)
2949	{
2950	u32 val, granularity;
2951	int ret;
2952
2953	if (!is_usb4_dpin(port))
2954	return -EOPNOTSUPP;
2955
2956	ret = usb4_dp_port_granularity(port);
2957	if (ret < 0)
2958	return ret;
2959	granularity = ret;
2960
2961	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2962	offset: port->cap_adap + DP_STATUS, length: 1);
2963	if (ret)
2964	return ret;
2965
2966	val &= DP_STATUS_ALLOCATED_BW_MASK;
2967	val >>= DP_STATUS_ALLOCATED_BW_SHIFT;
2968
2969	return val * granularity;
2970	}
2971
2972	static int __usb4_dp_port_set_cm_ack(struct tb_port *port, bool ack)
2973	{
2974	u32 val;
2975	int ret;
2976
2977	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
2978	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2979	if (ret)
2980	return ret;
2981
2982	if (ack)
2983	val |= ADP_DP_CS_2_CA;
2984	else
2985	val &= ~ADP_DP_CS_2_CA;
2986
2987	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
2988	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
2989	}
2990
2991	static inline int usb4_dp_port_set_cm_ack(struct tb_port *port)
2992	{
2993	return __usb4_dp_port_set_cm_ack(port, ack: true);
2994	}
2995
2996	static int usb4_dp_port_wait_and_clear_cm_ack(struct tb_port *port,
2997	int timeout_msec)
2998	{
2999	ktime_t end;
3000	u32 val;
3001	int ret;
3002
3003	ret = __usb4_dp_port_set_cm_ack(port, ack: false);
3004	if (ret)
3005	return ret;
3006
3007	end = ktime_add_ms(kt: ktime_get(), msec: timeout_msec);
3008	do {
3009	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
3010	offset: port->cap_adap + ADP_DP_CS_8, length: 1);
3011	if (ret)
3012	return ret;
3013
3014	if (!(val & ADP_DP_CS_8_DR))
3015	break;
3016
3017	usleep_range(min: 50, max: 100);
3018	} while (ktime_before(cmp1: ktime_get(), cmp2: end));
3019
3020	if (val & ADP_DP_CS_8_DR) {
3021	tb_port_warn(port, "timeout waiting for DPTX request to clear\n");
3022	return -ETIMEDOUT;
3023	}
3024
3025	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
3026	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
3027	if (ret)
3028	return ret;
3029
3030	val &= ~ADP_DP_CS_2_CA;
3031	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
3032	offset: port->cap_adap + ADP_DP_CS_2, length: 1);
3033	}
3034
3035	/**
3036	* usb4_dp_port_allocate_bandwidth() - Set allocated bandwidth
3037	* @port: DP IN adapter
3038	* @bw: New allocated bandwidth in Mb/s
3039	*
3040	* Communicates the new allocated bandwidth with the DPCD (graphics
3041	* driver). Takes into account the programmed granularity.
3042	*
3043	* Return: %0 on success, negative errno otherwise.
3044	*/
3045	int usb4_dp_port_allocate_bandwidth(struct tb_port *port, int bw)
3046	{
3047	u32 val, granularity;
3048	int ret;
3049
3050	if (!is_usb4_dpin(port))
3051	return -EOPNOTSUPP;
3052
3053	ret = usb4_dp_port_granularity(port);
3054	if (ret < 0)
3055	return ret;
3056	granularity = ret;
3057
3058	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
3059	offset: port->cap_adap + DP_STATUS, length: 1);
3060	if (ret)
3061	return ret;
3062
3063	val &= ~DP_STATUS_ALLOCATED_BW_MASK;
3064	val |= (bw / granularity) << DP_STATUS_ALLOCATED_BW_SHIFT;
3065
3066	ret = tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
3067	offset: port->cap_adap + DP_STATUS, length: 1);
3068	if (ret)
3069	return ret;
3070
3071	ret = usb4_dp_port_set_cm_ack(port);
3072	if (ret)
3073	return ret;
3074
3075	return usb4_dp_port_wait_and_clear_cm_ack(port, timeout_msec: 500);
3076	}
3077
3078	/**
3079	* usb4_dp_port_requested_bandwidth() - Read requested bandwidth
3080	* @port: DP IN adapter
3081	*
3082	* Reads the DPCD (graphics driver) requested bandwidth and returns it
3083	* in Mb/s. Takes the programmed granularity into account.
3084	*
3085	* Return:
3086	* * Requested bandwidth in Mb/s - On success.
3087	* * %-EOPNOTSUPP - If the adapter does not support bandwidth allocation
3088	* mode.
3089	* * %ENODATA - If there is no active bandwidth request from the graphics
3090	* driver.
3091	* * Negative errno - On failure.
3092	*/
3093	int usb4_dp_port_requested_bandwidth(struct tb_port *port)
3094	{
3095	u32 val, granularity;
3096	int ret;
3097
3098	if (!is_usb4_dpin(port))
3099	return -EOPNOTSUPP;
3100
3101	ret = usb4_dp_port_granularity(port);
3102	if (ret < 0)
3103	return ret;
3104	granularity = ret;
3105
3106	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
3107	offset: port->cap_adap + ADP_DP_CS_8, length: 1);
3108	if (ret)
3109	return ret;
3110
3111	if (!(val & ADP_DP_CS_8_DR))
3112	return -ENODATA;
3113
3114	return (val & ADP_DP_CS_8_REQUESTED_BW_MASK) * granularity;
3115	}
3116
3117	/**
3118	* usb4_pci_port_set_ext_encapsulation() - Enable/disable extended encapsulation
3119	* @port: PCIe adapter
3120	* @enable: Enable/disable extended encapsulation
3121	*
3122	* Enables or disables extended encapsulation used in PCIe tunneling. Caller
3123	* needs to make sure both adapters support this before enabling.
3124	*
3125	* Return: %0 on success, negative errno otherwise.
3126	*/
3127	int usb4_pci_port_set_ext_encapsulation(struct tb_port *port, bool enable)
3128	{
3129	u32 val;
3130	int ret;
3131
3132	if (!tb_port_is_pcie_up(port) && !tb_port_is_pcie_down(port))
3133	return -EINVAL;
3134
3135	ret = tb_port_read(port, buffer: &val, space: TB_CFG_PORT,
3136	offset: port->cap_adap + ADP_PCIE_CS_1, length: 1);
3137	if (ret)
3138	return ret;
3139
3140	if (enable)
3141	val |= ADP_PCIE_CS_1_EE;
3142	else
3143	val &= ~ADP_PCIE_CS_1_EE;
3144
3145	return tb_port_write(port, buffer: &val, space: TB_CFG_PORT,
3146	offset: port->cap_adap + ADP_PCIE_CS_1, length: 1);
3147	}
3148